You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by tu...@apache.org on 2020/10/04 11:51:27 UTC
[airflow] branch v1-10-test updated: Add ImportChangesRule for
upgrade check (#11056)
This is an automated email from the ASF dual-hosted git repository.
turbaszek pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v1-10-test by this push:
new 4431c92 Add ImportChangesRule for upgrade check (#11056)
4431c92 is described below
commit 4431c921e359b5cedfba5978df06c002afe5fb14
Author: Tomek Urbaszek <tu...@gmail.com>
AuthorDate: Sun Oct 4 13:48:16 2020 +0200
Add ImportChangesRule for upgrade check (#11056)
Adds new rule that will be run during airflow upgrade-check
and will look for old clasess and imports incompatible with
Airflow 2.0.
---
airflow/upgrade/rules/conn_type_is_not_nullable.py | 8 +-
airflow/upgrade/rules/import_changes.py | 89 +
airflow/upgrade/rules/renamed_classes.py | 1732 ++++++++++++++++++++
tests/upgrade/rules/test_import_changes.py | 68 +
4 files changed, 1895 insertions(+), 2 deletions(-)
diff --git a/airflow/upgrade/rules/conn_type_is_not_nullable.py b/airflow/upgrade/rules/conn_type_is_not_nullable.py
index 2ddd7c2..8f574d9 100644
--- a/airflow/upgrade/rules/conn_type_is_not_nullable.py
+++ b/airflow/upgrade/rules/conn_type_is_not_nullable.py
@@ -35,8 +35,12 @@ If you made any modifications to the table directly, make sure you don't have nu
@provide_session
def check(self, session=None):
- invalid_connections = session.query(Connection).filter(Connection.conn_type.is_(None))
+ invalid_connections = session.query(Connection).filter(
+ Connection.conn_type.is_(None)
+ )
return (
- 'Connection<id={}", conn_id={}> have empty conn_type field.'.format(conn.id, conn.conn_id)
+ 'Connection<id={}", conn_id={}> have empty conn_type field.'.format(
+ conn.id, conn.conn_id
+ )
for conn in invalid_connections
)
diff --git a/airflow/upgrade/rules/import_changes.py b/airflow/upgrade/rules/import_changes.py
new file mode 100644
index 0000000..f75867b
--- /dev/null
+++ b/airflow/upgrade/rules/import_changes.py
@@ -0,0 +1,89 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import NamedTuple, Optional, List
+
+from cached_property import cached_property
+
+from airflow import conf
+from airflow.upgrade.rules.base_rule import BaseRule
+from airflow.upgrade.rules.renamed_classes import ALL
+from airflow.utils.dag_processing import list_py_file_paths
+
+
+class ImportChange(
+ NamedTuple(
+ "ImportChange",
+ [("old_path", str), ("new_path", str), ("providers_package", Optional[None])],
+ )
+):
+ def info(self, file_path=None):
+ msg = "Using `{}` will be replaced by `{}`".format(self.old_path, self.new_path)
+ if self.providers_package:
+ msg += " and requires `{}` providers package".format(
+ self.providers_package
+ )
+ if file_path:
+ msg += ". Affected file: {}".format(file_path)
+ return msg
+
+ @cached_property
+ def old_class(self):
+ return self.old_path.split(".")[-1]
+
+ @cached_property
+ def new_class(self):
+ return self.new_path.split(".")[-1]
+
+ @classmethod
+ def from_new_old_paths(cls, new_path, old_path):
+ providers_package = new_path.split(".")[2] if "providers" in new_path else None
+ return cls(
+ old_path=old_path, new_path=new_path, providers_package=providers_package
+ )
+
+
+class ImportChangesRule(BaseRule):
+ title = "Changes in import paths of hooks, operators, sensors and others"
+ description = (
+ "Many hooks, operators and other classes has been renamed and moved. Those changes were part of "
+ "unifying names and imports paths as described in AIP-21.\nThe `contrib` folder has been replaced "
+ "by `providers` directory and packages:\n"
+ "https://github.com/apache/airflow#backport-packages"
+ )
+
+ ALL_CHANGES = [
+ ImportChange.from_new_old_paths(*args) for args in ALL
+ ] # type: List[ImportChange]
+
+ @staticmethod
+ def _check_file(file_path):
+ problems = []
+ with open(file_path, "r") as file:
+ content = file.read()
+ for change in ImportChangesRule.ALL_CHANGES:
+ if change.old_class in content:
+ problems.append(change.info(file_path))
+ return problems
+
+ def check(self):
+ dag_folder = conf.get("core", "dags_folder")
+ files = list_py_file_paths(directory=dag_folder, include_examples=False)
+ problems = []
+ for file in files:
+ problems.extend(self._check_file(file))
+ return problems
diff --git a/airflow/upgrade/rules/renamed_classes.py b/airflow/upgrade/rules/renamed_classes.py
new file mode 100644
index 0000000..00227b0
--- /dev/null
+++ b/airflow/upgrade/rules/renamed_classes.py
@@ -0,0 +1,1732 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+HOOKS = [
+ (
+ "airflow.providers.apache.cassandra.hooks.cassandra.CassandraHook",
+ "airflow.contrib.hooks.cassandra_hook.CassandraHook",
+ ),
+ (
+ "airflow.providers.google.cloud.hooks.compute.ComputeEngineHook",
+ "airflow.contrib.hooks.gcp_compute_hook.GceHook",
+ ),
+ (
+ "airflow.providers.google.common.hooks.base_google.GoogleBaseHook",
+ "airflow.contrib.hooks.gcp_api_base_hook.GoogleBaseHook",
+ ),
+ (
+ "airflow.providers.google.cloud.hooks.dataflow.DataflowHook",
+ "airflow.contrib.hooks.gcp_dataflow_hook.DataFlowHook",
+ ),
+ (
+ "airflow.providers.google.cloud.hooks.dataproc.DataprocHook",
+ "airflow.contrib.hooks.gcp_dataproc_hook.DataProcHook",
+ ),
+ (
+ "airflow.providers.google.cloud.hooks.dlp.CloudDLPHook",
+ "airflow.contrib.hooks.gcp_dlp_hook.CloudDLPHook",
+ ),
+ (
+ "airflow.providers.google.cloud.hooks.functions.CloudFunctionsHook",
+ "airflow.contrib.hooks.gcp_function_hook.GcfHook",
+ ),
+ (
+ "airflow.providers.google.cloud.hooks.kms.CloudKMSHook",
+ "airflow.contrib.hooks.gcp_kms_hook.GoogleCloudKMSHook",
+ ),
+ (
+ "airflow.providers.google.cloud.hooks.mlengine.MLEngineHook",
+ "airflow.contrib.hooks.gcp_mlengine_hook.MLEngineHook",
+ ),
+ (
+ "airflow.providers.google.cloud.hooks.spanner.SpannerHook",
+ "airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook",
+ ),
+ (
+ "airflow.providers.google.cloud.hooks.speech_to_text.CloudSpeechToTextHook",
+ "airflow.contrib.hooks.gcp_speech_to_text_hook.GCPSpeechToTextHook",
+ ),
+ (
+ "airflow.providers.google.cloud.hooks.text_to_speech.CloudTextToSpeechHook",
+ "airflow.contrib.hooks.gcp_text_to_speech_hook.GCPTextToSpeechHook",
+ ),
+ (
+ "airflow.providers.google.cloud.hooks.gcs.GCSHook",
+ "airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook",
+ ),
+ (
+ "airflow.providers.google.cloud.hooks.cloud_build.CloudBuildHook",
+ "airflow.contrib.hooks.gcp_cloud_build_hook.CloudBuildHook",
+ ),
+ (
+ "airflow.providers.google.cloud.hooks.bigtable.BigtableHook",
+ "airflow.contrib.hooks.gcp_bigtable_hook.BigtableHook",
+ ),
+ (
+ "airflow.providers.google.cloud.hooks.kubernetes_engine.GKEHook",
+ "airflow.contrib.hooks.gcp_container_hook.GKEClusterHook",
+ ),
+ (
+ "airflow.providers.google.cloud.hooks.datastore.DatastoreHook",
+ "airflow.contrib.hooks.datastore_hook.DatastoreHook",
+ ),
+ (
+ "airflow.providers.google.cloud.hooks.natural_language.CloudNaturalLanguageHook",
+ "airflow.contrib.hooks.gcp_natural_language_hook.CloudNaturalLanguageHook",
+ ),
+ (
+ "airflow.providers.google.cloud.hooks.pubsub.PubSubHook",
+ "airflow.contrib.hooks.gcp_pubsub_hook.PubSubHook",
+ ),
+ (
+ "airflow.providers.google.cloud.hooks.cloud_sql.CloudSQLHook",
+ "airflow.contrib.hooks.gcp_sql_hook.CloudSqlHook",
+ ),
+ (
+ "airflow.providers.google.cloud.hooks.cloud_sql.CloudSQLDatabaseHook",
+ "airflow.contrib.hooks.gcp_sql_hook.CloudSqlDatabaseHook",
+ ),
+ (
+ "airflow.providers.google.cloud.hooks.tasks.CloudTasksHook",
+ "airflow.contrib.hooks.gcp_tasks_hook.CloudTasksHook",
+ ),
+ (
+ "airflow.providers.google.cloud.hooks.cloud_storage_transfer_service.CloudDataTransferServiceHook",
+ "airflow.contrib.hooks.gcp_transfer_hook.GCPTransferServiceHook",
+ ),
+ (
+ "airflow.providers.google.cloud.hooks.translate.CloudTranslateHook",
+ "airflow.contrib.hooks.gcp_translate_hook.CloudTranslateHook",
+ ),
+ (
+ "airflow.providers.google.cloud.hooks.video_intelligence.CloudVideoIntelligenceHook",
+ "airflow.contrib.hooks.gcp_video_intelligence_hook.CloudVideoIntelligenceHook",
+ ),
+ (
+ "airflow.providers.google.cloud.hooks.vision.CloudVisionHook",
+ "airflow.contrib.hooks.gcp_vision_hook.CloudVisionHook",
+ ),
+ (
+ "airflow.providers.google.cloud.hooks.bigquery.BigQueryHook",
+ "airflow.contrib.hooks.bigquery_hook.BigQueryHook",
+ ),
+ (
+ "airflow.providers.amazon.aws.hooks.athena.AWSAthenaHook",
+ "airflow.contrib.hooks.aws_athena_hook.AWSAthenaHook",
+ ),
+ (
+ "airflow.providers.amazon.aws.hooks.datasync.AWSDataSyncHook",
+ "airflow.contrib.hooks.aws_datasync_hook.AWSDataSyncHook",
+ ),
+ ("airflow.providers.amazon.aws.hooks.s3.S3Hook", "airflow.hooks.S3_hook.S3Hook"),
+ (
+ "airflow.providers.amazon.aws.hooks.sqs.SQSHook",
+ "airflow.contrib.hooks.aws_sqs_hook.SQSHook",
+ ),
+ (
+ "airflow.providers.amazon.aws.hooks.lambda_function.AwsLambdaHook",
+ "airflow.contrib.hooks.aws_lambda_hook.AwsLambdaHook",
+ ),
+ (
+ "airflow.providers.amazon.aws.hooks.sns.AwsSnsHook",
+ "airflow.contrib.hooks.aws_sns_hook.AwsSnsHook",
+ ),
+ (
+ "airflow.providers.apache.pinot.hooks.pinot.PinotDbApiHook",
+ "airflow.contrib.hooks.pinot_hook.PinotDbApiHook",
+ ),
+ (
+ "airflow.providers.apache.pinot.hooks.pinot.PinotAdminHook",
+ "airflow.contrib.hooks.pinot_hook.PinotAdminHook",
+ ),
+ (
+ "airflow.providers.apache.spark.hooks.spark_jdbc.SparkJDBCHook",
+ "airflow.contrib.hooks.spark_jdbc_hook.SparkJDBCHook",
+ ),
+ (
+ "airflow.providers.apache.spark.hooks.spark_sql.SparkSqlHook",
+ "airflow.contrib.hooks.spark_sql_hook.SparkSqlHook",
+ ),
+ (
+ "airflow.providers.apache.spark.hooks.spark_submit.SparkSubmitHook",
+ "airflow.contrib.hooks.spark_submit_hook.SparkSubmitHook",
+ ),
+ (
+ "airflow.providers.apache.sqoop.hooks.sqoop.SqoopHook",
+ "airflow.contrib.hooks.sqoop_hook.SqoopHook",
+ ),
+ (
+ "airflow.providers.apache.druid.hooks.druid.DruidHook",
+ "airflow.hooks.druid_hook.DruidHook",
+ ),
+ (
+ "airflow.providers.apache.druid.hooks.druid.DruidDbApiHook",
+ "airflow.hooks.druid_hook.DruidDbApiHook",
+ ),
+ (
+ "airflow.providers.apache.hdfs.hooks.hdfs.HDFSHookException",
+ "airflow.hooks.hdfs_hook.HDFSHookException",
+ ),
+ (
+ "airflow.providers.apache.hdfs.hooks.hdfs.HDFSHook",
+ "airflow.hooks.hdfs_hook.HDFSHook",
+ ),
+ (
+ "airflow.providers.apache.hive.hooks.hive.HiveMetastoreHook",
+ "airflow.hooks.hive_hooks.HiveMetastoreHook",
+ ),
+ (
+ "airflow.providers.apache.hive.hooks.hive.HiveCliHook",
+ "airflow.hooks.hive_hooks.HiveCliHook",
+ ),
+ (
+ "airflow.providers.apache.hive.hooks.hive.HiveServer2Hook",
+ "airflow.hooks.hive_hooks.HiveServer2Hook",
+ ),
+ (
+ "airflow.providers.apache.pig.hooks.pig.PigCliHook",
+ "airflow.hooks.pig_hook.PigCliHook",
+ ),
+ (
+ "airflow.providers.apache.hdfs.hooks.webhdfs.WebHDFSHook",
+ "airflow.hooks.webhdfs_hook.WebHDFSHook",
+ ),
+ ("airflow.hooks.filesystem.FSHook", "airflow.contrib.hooks.fs_hook.FSHook"),
+ (
+ "airflow.providers.microsoft.azure.hooks.azure_container_instance.AzureContainerInstanceHook",
+ "airflow.contrib.hooks.azure_container_instance_hook.AzureContainerInstanceHook",
+ ),
+ (
+ "airflow.providers.microsoft.azure.hooks.azure_container_registry.AzureContainerRegistryHook",
+ "airflow.contrib.hooks.azure_container_registry_hook.AzureContainerRegistryHook",
+ ),
+ (
+ "airflow.providers.microsoft.azure.hooks.azure_container_volume.AzureContainerVolumeHook",
+ "airflow.contrib.hooks.azure_container_volume_hook.AzureContainerVolumeHook",
+ ),
+ (
+ "airflow.providers.microsoft.azure.hooks.azure_cosmos.AzureCosmosDBHook",
+ "airflow.contrib.hooks.azure_cosmos_hook.AzureCosmosDBHook",
+ ),
+ (
+ "airflow.providers.microsoft.azure.hooks.azure_fileshare.AzureFileShareHook",
+ "airflow.contrib.hooks.azure_fileshare_hook.AzureFileShareHook",
+ ),
+ (
+ "airflow.providers.microsoft.azure.hooks.wasb.WasbHook",
+ "airflow.contrib.hooks.wasb_hook.WasbHook",
+ ),
+ (
+ "airflow.providers.amazon.aws.hooks.glue_catalog.AwsGlueCatalogHook",
+ "airflow.contrib.hooks.aws_glue_catalog_hook.AwsGlueCatalogHook",
+ ),
+ (
+ "airflow.providers.amazon.aws.hooks.logs.AwsLogsHook",
+ "airflow.contrib.hooks.aws_logs_hook.AwsLogsHook",
+ ),
+ (
+ "airflow.providers.amazon.aws.hooks.emr.EmrHook",
+ "airflow.contrib.hooks.emr_hook.EmrHook",
+ ),
+ (
+ "airflow.providers.amazon.aws.hooks.sagemaker.SageMakerHook",
+ "airflow.contrib.hooks.sagemaker_hook.SageMakerHook",
+ ),
+ (
+ "airflow.providers.mongo.hooks.mongo.MongoHook",
+ "airflow.contrib.hooks.mongo_hook.MongoHook",
+ ),
+ (
+ "airflow.providers.openfaas.hooks.openfaas.OpenFaasHook",
+ "airflow.contrib.hooks.openfaas_hook.OpenFaasHook",
+ ),
+ (
+ "airflow.providers.redis.hooks.redis.RedisHook",
+ "airflow.contrib.hooks.redis_hook.RedisHook",
+ ),
+ (
+ "airflow.providers.docker.hooks.docker.DockerHook",
+ "airflow.hooks.docker_hook.DockerHook",
+ ),
+ (
+ "airflow.providers.microsoft.mssql.hooks.mssql.MsSqlHook",
+ "airflow.hooks.mssql_hook.MsSqlHook",
+ ),
+ (
+ "airflow.providers.mysql.hooks.mysql.MySqlHook",
+ "airflow.hooks.mysql_hook.MySqlHook",
+ ),
+ (
+ "airflow.providers.oracle.hooks.oracle.OracleHook",
+ "airflow.hooks.oracle_hook.OracleHook",
+ ),
+ (
+ "airflow.providers.postgres.hooks.postgres.PostgresHook",
+ "airflow.hooks.postgres_hook.PostgresHook",
+ ),
+ (
+ "airflow.providers.presto.hooks.presto.PrestoHook",
+ "airflow.hooks.presto_hook.PrestoHook",
+ ),
+ (
+ "airflow.providers.samba.hooks.samba.SambaHook",
+ "airflow.hooks.samba_hook.SambaHook",
+ ),
+ (
+ "airflow.providers.sqlite.hooks.sqlite.SqliteHook",
+ "airflow.hooks.sqlite_hook.SqliteHook",
+ ),
+ (
+ "airflow.providers.cloudant.hooks.cloudant.CloudantHook",
+ "airflow.contrib.hooks.cloudant_hook.CloudantHook",
+ ),
+ (
+ "airflow.providers.databricks.hooks.databricks.DatabricksHook",
+ "airflow.contrib.hooks.databricks_hook.DatabricksHook",
+ ),
+ (
+ "airflow.providers.databricks.hooks.databricks.DatabricksHook",
+ "airflow.contrib.hooks.databricks_hook.DatabricksHook",
+ ),
+ (
+ "airflow.providers.datadog.hooks.datadog.DatadogHook",
+ "airflow.contrib.hooks.datadog_hook.DatadogHook",
+ ),
+ (
+ "airflow.providers.dingding.hooks.dingding.DingdingHook",
+ "airflow.contrib.hooks.dingding_hook.DingdingHook",
+ ),
+ (
+ "airflow.providers.discord.hooks.discord_webhook.DiscordWebhookHook",
+ "airflow.contrib.hooks.discord_webhook_hook.DiscordWebhookHook",
+ ),
+ (
+ "airflow.providers.google.suite.hooks.drive.GoogleDriveHook",
+ "airflow.contrib.hooks.gdrive_hook.GoogleDriveHook",
+ ),
+ (
+ "airflow.providers.jenkins.hooks.jenkins.JenkinsHook",
+ "airflow.contrib.hooks.jenkins_hook.JenkinsHook",
+ ),
+ (
+ "airflow.providers.jira.hooks.jira.JiraHook",
+ "airflow.contrib.hooks.jira_hook.JiraHook",
+ ),
+ (
+ "airflow.providers.opsgenie.hooks.opsgenie_alert.OpsgenieAlertHook",
+ "airflow.contrib.hooks.opsgenie_alert_hook.OpsgenieAlertHook",
+ ),
+ (
+ "airflow.providers.pagerduty.hooks.pagerduty.PagerdutyHook",
+ "airflow.contrib.hooks.pagerduty_hook.PagerdutyHook",
+ ),
+ (
+ "airflow.providers.qubole.hooks.qubole_check.QuboleCheckHook",
+ "airflow.contrib.hooks.qubole_check_hook.QuboleCheckHook",
+ ),
+ (
+ "airflow.providers.qubole.hooks.qubole.QuboleHook",
+ "airflow.contrib.hooks.qubole_hook.QuboleHook",
+ ),
+ (
+ "airflow.providers.salesforce.hooks.salesforce.SalesforceHook",
+ "airflow.contrib.hooks.salesforce_hook.SalesforceHook",
+ ),
+ (
+ "airflow.providers.segment.hooks.segment.SegmentHook",
+ "airflow.contrib.hooks.segment_hook.SegmentHook",
+ ),
+ (
+ "airflow.providers.slack.hooks.slack_webhook.SlackWebhookHook",
+ "airflow.contrib.hooks.slack_webhook_hook.SlackWebhookHook",
+ ),
+ (
+ "airflow.providers.vertica.hooks.vertica.VerticaHook",
+ "airflow.contrib.hooks.vertica_hook.VerticaHook",
+ ),
+ (
+ "airflow.providers.slack.hooks.slack.SlackHook",
+ "airflow.hooks.slack_hook.SlackHook",
+ ),
+ (
+ "airflow.providers.zendesk.hooks.zendesk.ZendeskHook",
+ "airflow.hooks.zendesk_hook.ZendeskHook",
+ ),
+ (
+ "airflow.providers.ftp.hooks.ftp.FTPSHook",
+ "airflow.contrib.hooks.ftp_hook.FTPSHook",
+ ),
+ (
+ "airflow.providers.ftp.hooks.ftp.FTPHook",
+ "airflow.contrib.hooks.ftp_hook.FTPHook",
+ ),
+ (
+ "airflow.providers.imap.hooks.imap.ImapHook",
+ "airflow.contrib.hooks.imap_hook.ImapHook",
+ ),
+ (
+ "airflow.providers.ssh.hooks.ssh.SSHHook",
+ "airflow.contrib.hooks.ssh_hook.SSHHook",
+ ),
+ (
+ "airflow.providers.microsoft.winrm.hooks.winrm.WinRMHook",
+ "airflow.contrib.hooks.winrm_hook.WinRMHook",
+ ),
+ ("airflow.providers.http.hooks.http.HttpHook", "airflow.hooks.http_hook.HttpHook"),
+ ("airflow.providers.jdbc.hooks.jdbc.JdbcHook", "airflow.hooks.jdbc_hook.JdbcHook"),
+ (
+ "airflow.providers.amazon.aws.hooks.base_aws.AwsBaseHook",
+ "airflow.contrib.hooks.aws_hook.AwsHook",
+ ),
+ (
+ "airflow.providers.amazon.aws.hooks.aws_dynamodb.AwsDynamoDBHook",
+ "airflow.contrib.hooks.aws_dynamodb_hook.AwsDynamoDBHook",
+ ),
+ (
+ "airflow.providers.sftp.hooks.sftp.SFTPHook",
+ "airflow.contrib.hooks.sftp_hook.SFTPHook",
+ ),
+]
+
+OPERATORS = [
+ (
+ "airflow.providers.google.cloud.operators.dataflow.DataflowCreateJavaJobOperator",
+ "airflow.contrib.operators.dataflow_operator.DataFlowJavaOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dataflow.DataflowCreatePythonJobOperator",
+ "airflow.contrib.operators.dataflow_operator.DataFlowPythonOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dataflow.DataflowTemplatedJobStartOperator",
+ "airflow.contrib.operators.dataflow_operator.DataflowTemplateOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.datastore.CloudDatastoreExportEntitiesOperator",
+ "airflow.contrib.operators.datastore_export_operator.DatastoreExportOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.datastore.CloudDatastoreImportEntitiesOperator",
+ "airflow.contrib.operators.datastore_import_operator.DatastoreImportOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.bigtable.BigtableUpdateClusterOperator",
+ "airflow.contrib.operators.gcp_bigtable_operator.BigtableClusterUpdateOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.bigtable.BigtableCreateInstanceOperator",
+ "airflow.contrib.operators.gcp_bigtable_operator.BigtableInstanceCreateOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.bigtable.BigtableDeleteInstanceOperator",
+ "airflow.contrib.operators.gcp_bigtable_operator.BigtableInstanceDeleteOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.bigtable.BigtableCreateTableOperator",
+ "airflow.contrib.operators.gcp_bigtable_operator.BigtableTableCreateOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.bigtable.BigtableDeleteTableOperator",
+ "airflow.contrib.operators.gcp_bigtable_operator.BigtableTableDeleteOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.cloud_build.CloudBuildCreateBuildOperator",
+ "airflow.contrib.operators.gcp_cloud_build_operator.CloudBuildCreateBuildOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.compute.ComputeEngineBaseOperator",
+ "airflow.contrib.operators.gcp_compute_operator.GceBaseOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.compute"
+ ".ComputeEngineInstanceGroupUpdateManagerTemplateOperator",
+ "airflow.contrib.operators.gcp_compute_operator."
+ "GceInstanceGroupManagerUpdateTemplateOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.compute.ComputeEngineStartInstanceOperator",
+ "airflow.contrib.operators.gcp_compute_operator.GceInstanceStartOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.compute.ComputeEngineStopInstanceOperator",
+ "airflow.contrib.operators.gcp_compute_operator.GceInstanceStopOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.compute.ComputeEngineCopyInstanceTemplateOperator",
+ "airflow.contrib.operators.gcp_compute_operator.GceInstanceTemplateCopyOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.compute.ComputeEngineSetMachineTypeOperator",
+ "airflow.contrib.operators.gcp_compute_operator.GceSetMachineTypeOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.kubernetes_engine.GKECreateClusterOperator",
+ "airflow.contrib.operators.gcp_container_operator.GKEClusterCreateOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.kubernetes_engine.GKEDeleteClusterOperator",
+ "airflow.contrib.operators.gcp_container_operator.GKEClusterDeleteOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.kubernetes_engine.GKEStartPodOperator",
+ "airflow.contrib.operators.gcp_container_operator.GKEPodOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPCancelDLPJobOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPCancelDLPJobOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPCreateDeidentifyTemplateOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPCreateDeidentifyTemplateOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPCreateDLPJobOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPCreateDLPJobOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPCreateInspectTemplateOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPCreateInspectTemplateOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPCreateJobTriggerOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPCreateJobTriggerOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPCreateStoredInfoTypeOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPCreateStoredInfoTypeOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPDeidentifyContentOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPDeidentifyContentOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPDeleteDeidentifyTemplateOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPDeleteDeidentifyTemplateOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPDeleteDLPJobOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPDeleteDlpJobOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPDeleteInspectTemplateOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPDeleteInspectTemplateOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPDeleteJobTriggerOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPDeleteJobTriggerOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPDeleteStoredInfoTypeOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPDeleteStoredInfoTypeOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPGetDeidentifyTemplateOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPGetDeidentifyTemplateOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPGetDLPJobOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPGetDlpJobOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPGetInspectTemplateOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPGetInspectTemplateOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPGetDLPJobTriggerOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPGetJobTripperOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPGetStoredInfoTypeOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPGetStoredInfoTypeOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPInspectContentOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPInspectContentOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPListDeidentifyTemplatesOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPListDeidentifyTemplatesOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPListDLPJobsOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPListDlpJobsOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPListInfoTypesOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPListInfoTypesOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPListInspectTemplatesOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPListInspectTemplatesOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPListJobTriggersOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPListJobTriggersOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPListStoredInfoTypesOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPListStoredInfoTypesOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPRedactImageOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPRedactImageOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPReidentifyContentOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPReidentifyContentOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPUpdateDeidentifyTemplateOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPUpdateDeidentifyTemplateOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPUpdateInspectTemplateOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPUpdateInspectTemplateOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPUpdateJobTriggerOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPUpdateJobTriggerOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.dlp.CloudDLPUpdateStoredInfoTypeOperator",
+ "airflow.contrib.operators.gcp_dlp_operator.CloudDLPUpdateStoredInfoTypeOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.functions.CloudFunctionDeleteFunctionOperator",
+ "airflow.contrib.operators.gcp_function_operator.GcfFunctionDeleteOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.functions.CloudFunctionDeployFunctionOperator",
+ "airflow.contrib.operators.gcp_function_operator.GcfFunctionDeployOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.natural_language."
+ "CloudNaturalLanguageAnalyzeEntitiesOperator",
+ "airflow.contrib.operators.gcp_natural_language_operator."
+ "CloudLanguageAnalyzeEntitiesOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.natural_language."
+ "CloudNaturalLanguageAnalyzeEntitySentimentOperator",
+ "airflow.contrib.operators.gcp_natural_language_operator."
+ "CloudLanguageAnalyzeEntitySentimentOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.natural_language."
+ "CloudNaturalLanguageAnalyzeSentimentOperator",
+ "airflow.contrib.operators.gcp_natural_language_operator."
+ "CloudLanguageAnalyzeSentimentOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.natural_language."
+ "CloudNaturalLanguageClassifyTextOperator",
+ "airflow.contrib.operators.gcp_natural_language_operator.CloudLanguageClassifyTextOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.spanner.SpannerDeleteDatabaseInstanceOperator",
+ "airflow.contrib.operators.gcp_spanner_operator.CloudSpannerInstanceDatabaseDeleteOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.spanner.SpannerDeployDatabaseInstanceOperator",
+ "airflow.contrib.operators.gcp_spanner_operator.CloudSpannerInstanceDatabaseDeployOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.spanner.SpannerQueryDatabaseInstanceOperator",
+ "airflow.contrib.operators.gcp_spanner_operator.CloudSpannerInstanceDatabaseQueryOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.spanner.SpannerUpdateDatabaseInstanceOperator",
+ "airflow.contrib.operators.gcp_spanner_operator.CloudSpannerInstanceDatabaseUpdateOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.spanner.SpannerDeleteInstanceOperator",
+ "airflow.contrib.operators.gcp_spanner_operator.CloudSpannerInstanceDeleteOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.spanner.SpannerDeployInstanceOperator",
+ "airflow.contrib.operators.gcp_spanner_operator.CloudSpannerInstanceDeployOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.cloud_storage_transfer_service"
+ ".CloudDataTransferServiceCreateJobOperator",
+ "airflow.contrib.operators.gcp_transfer_operator.GcpTransferServiceJobCreateOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.cloud_storage_transfer_service"
+ ".CloudDataTransferServiceDeleteJobOperator",
+ "airflow.contrib.operators.gcp_transfer_operator.GcpTransferServiceJobDeleteOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.cloud_storage_transfer_service"
+ ".CloudDataTransferServiceUpdateJobOperator",
+ "airflow.contrib.operators.gcp_transfer_operator.GcpTransferServiceJobUpdateOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.cloud_storage_transfer_service."
+ "CloudDataTransferServiceCancelOperationOperator",
+ "airflow.contrib.operators.gcp_transfer_operator."
+ "GcpTransferServiceOperationCancelOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.cloud_storage_transfer_service."
+ "CloudDataTransferServiceGetOperationOperator",
+ "airflow.contrib.operators.gcp_transfer_operator."
+ "GcpTransferServiceOperationGetOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.cloud_storage_transfer_service."
+ "CloudDataTransferServicePauseOperationOperator",
+ "airflow.contrib.operators.gcp_transfer_operator."
+ "GcpTransferServiceOperationPauseOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.cloud_storage_transfer_service."
+ "CloudDataTransferServiceResumeOperationOperator",
+ "airflow.contrib.operators.gcp_transfer_operator."
+ "GcpTransferServiceOperationResumeOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.cloud_storage_transfer_service."
+ "CloudDataTransferServiceListOperationsOperator",
+ "airflow.contrib.operators.gcp_transfer_operator."
+ "GcpTransferServiceOperationsListOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.translate.CloudTranslateTextOperator",
+ "airflow.contrib.operators.gcp_translate_operator.CloudTranslateTextOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.translate_speech.CloudTranslateSpeechOperator",
+ "airflow.contrib.operators.gcp_translate_speech_operator.CloudTranslateSpeechOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.video_intelligence."
+ "CloudVideoIntelligenceDetectVideoExplicitContentOperator",
+ "airflow.contrib.operators.gcp_video_intelligence_operator."
+ "CloudVideoIntelligenceDetectVideoExplicitContentOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.video_intelligence."
+ "CloudVideoIntelligenceDetectVideoLabelsOperator",
+ "airflow.contrib.operators.gcp_video_intelligence_operator."
+ "CloudVideoIntelligenceDetectVideoLabelsOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.video_intelligence."
+ "CloudVideoIntelligenceDetectVideoShotsOperator",
+ "airflow.contrib.operators.gcp_video_intelligence_operator."
+ "CloudVideoIntelligenceDetectVideoShotsOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.vision.CloudVisionImageAnnotateOperator",
+ "airflow.contrib.operators.gcp_vision_operator.CloudVisionAnnotateImageOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.vision.CloudVisionTextDetectOperator",
+ "airflow.contrib.operators.gcp_vision_operator.CloudVisionDetectDocumentTextOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.vision.CloudVisionDetectImageLabelsOperator",
+ "airflow.contrib.operators.gcp_vision_operator.CloudVisionDetectImageLabelsOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.vision.CloudVisionDetectImageSafeSearchOperator",
+ "airflow.contrib.operators.gcp_vision_operator.CloudVisionDetectImageSafeSearchOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.vision.CloudVisionDetectTextOperator",
+ "airflow.contrib.operators.gcp_vision_operator.CloudVisionDetectTextOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.vision.CloudVisionCreateProductOperator",
+ "airflow.contrib.operators.gcp_vision_operator.CloudVisionProductCreateOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.vision.CloudVisionDeleteProductOperator",
+ "airflow.contrib.operators.gcp_vision_operator.CloudVisionProductDeleteOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.vision.CloudVisionGetProductOperator",
+ "airflow.contrib.operators.gcp_vision_operator.CloudVisionProductGetOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.vision.CloudVisionCreateProductSetOperator",
+ "airflow.contrib.operators.gcp_vision_operator.CloudVisionProductSetCreateOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.vision.CloudVisionDeleteProductSetOperator",
+ "airflow.contrib.operators.gcp_vision_operator.CloudVisionProductSetDeleteOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.vision.CloudVisionGetProductSetOperator",
+ "airflow.contrib.operators.gcp_vision_operator.CloudVisionProductSetGetOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.vision.CloudVisionUpdateProductSetOperator",
+ "airflow.contrib.operators.gcp_vision_operator.CloudVisionProductSetUpdateOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.vision.CloudVisionUpdateProductOperator",
+ "airflow.contrib.operators.gcp_vision_operator.CloudVisionProductUpdateOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.vision.CloudVisionCreateReferenceImageOperator",
+ "airflow.contrib.operators.gcp_vision_operator.CloudVisionReferenceImageCreateOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.vision.CloudVisionRemoveProductFromProductSetOperator",
+ "airflow.contrib.operators.gcp_vision_operator."
+ "CloudVisionRemoveProductFromProductSetOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.mlengine.MLEngineStartBatchPredictionJobOperator",
+ "airflow.contrib.operators.mlengine_operator.MLEngineBatchPredictionOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.mlengine.MLEngineManageModelOperator",
+ "airflow.contrib.operators.mlengine_operator.MLEngineModelOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.mlengine.MLEngineStartTrainingJobOperator",
+ "airflow.contrib.operators.mlengine_operator.MLEngineTrainingOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.mlengine.MLEngineManageVersionOperator",
+ "airflow.contrib.operators.mlengine_operator.MLEngineVersionOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.pubsub.PubSubPublishMessageOperator",
+ "airflow.contrib.operators.pubsub_operator.PubSubPublishOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.pubsub.PubSubCreateSubscriptionOperator",
+ "airflow.contrib.operators.pubsub_operator.PubSubSubscriptionCreateOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.pubsub.PubSubDeleteSubscriptionOperator",
+ "airflow.contrib.operators.pubsub_operator.PubSubSubscriptionDeleteOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.pubsub.PubSubCreateTopicOperator",
+ "airflow.contrib.operators.pubsub_operator.PubSubTopicCreateOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.pubsub.PubSubDeleteTopicOperator",
+ "airflow.contrib.operators.pubsub_operator.PubSubTopicDeleteOperator",
+ ),
+ (
+ "airflow.providers.google.cloud."
+ "operators.dataproc.DataprocCreateClusterOperator",
+ "airflow.contrib.operators.dataproc_operator.DataprocClusterCreateOperator",
+ ),
+ (
+ "airflow.providers.google.cloud."
+ "operators.dataproc.DataprocDeleteClusterOperator",
+ "airflow.contrib.operators.dataproc_operator.DataprocClusterDeleteOperator",
+ ),
+ (
+ "airflow.providers.google.cloud."
+ "operators.dataproc.DataprocScaleClusterOperator",
+ "airflow.contrib.operators.dataproc_operator.DataprocClusterScaleOperator",
+ ),
+ (
+ "airflow.providers.google.cloud."
+ "operators.dataproc.DataprocSubmitHadoopJobOperator",
+ "airflow.contrib.operators.dataproc_operator.DataProcHadoopOperator",
+ ),
+ (
+ "airflow.providers.google.cloud."
+ "operators.dataproc.DataprocSubmitHiveJobOperator",
+ "airflow.contrib.operators.dataproc_operator.DataProcHiveOperator",
+ ),
+ (
+ "airflow.providers.google.cloud." "operators.dataproc.DataprocJobBaseOperator",
+ "airflow.contrib.operators.dataproc_operator.DataProcJobBaseOperator",
+ ),
+ (
+ "airflow.providers.google.cloud."
+ "operators.dataproc.DataprocSubmitPigJobOperator",
+ "airflow.contrib.operators.dataproc_operator.DataProcPigOperator",
+ ),
+ (
+ "airflow.providers.google.cloud."
+ "operators.dataproc.DataprocSubmitPySparkJobOperator",
+ "airflow.contrib.operators.dataproc_operator.DataProcPySparkOperator",
+ ),
+ (
+ "airflow.providers.google.cloud."
+ "operators.dataproc.DataprocSubmitSparkJobOperator",
+ "airflow.contrib.operators.dataproc_operator.DataProcSparkOperator",
+ ),
+ (
+ "airflow.providers.google.cloud."
+ "operators.dataproc.DataprocSubmitSparkSqlJobOperator",
+ "airflow.contrib.operators.dataproc_operator.DataProcSparkSqlOperator",
+ ),
+ (
+ "airflow.providers.google.cloud."
+ "operators.dataproc.DataprocInstantiateInlineWorkflowTemplateOperator",
+ "airflow.contrib.operators.dataproc_operator."
+ "DataprocWorkflowTemplateInstantiateInlineOperator",
+ ),
+ (
+ "airflow.providers.google.cloud."
+ "operators.dataproc.DataprocInstantiateWorkflowTemplateOperator",
+ "airflow.contrib.operators.dataproc_operator."
+ "DataprocWorkflowTemplateInstantiateOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.bigquery.BigQueryCreateEmptyDatasetOperator",
+ "airflow.contrib.operators.bigquery_operator.BigQueryCreateEmptyDatasetOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.bigquery.BigQueryCreateEmptyTableOperator",
+ "airflow.contrib.operators.bigquery_operator.BigQueryCreateEmptyTableOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.bigquery.BigQueryCreateExternalTableOperator",
+ "airflow.contrib.operators.bigquery_operator.BigQueryCreateExternalTableOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.bigquery.BigQueryDeleteDatasetOperator",
+ "airflow.contrib.operators.bigquery_operator.BigQueryDeleteDatasetOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.bigquery.BigQueryGetDatasetOperator",
+ "airflow.contrib.operators.bigquery_operator.BigQueryGetDatasetOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.bigquery.BigQueryGetDatasetTablesOperator",
+ "airflow.contrib.operators.bigquery_operator.BigQueryGetDatasetTablesOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.bigquery.BigQueryPatchDatasetOperator",
+ "airflow.contrib.operators.bigquery_operator.BigQueryPatchDatasetOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.bigquery.BigQueryUpdateDatasetOperator",
+ "airflow.contrib.operators.bigquery_operator.BigQueryUpdateDatasetOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.bigquery.BigQueryUpsertTableOperator",
+ "airflow.contrib.operators.bigquery_operator.BigQueryUpsertTableOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.bigquery.BigQueryCheckOperator",
+ "airflow.contrib.operators.bigquery_check_operator.BigQueryCheckOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.bigquery.BigQueryIntervalCheckOperator",
+ "airflow.contrib.operators.bigquery_check_operator.BigQueryIntervalCheckOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.bigquery.BigQueryValueCheckOperator",
+ "airflow.contrib.operators.bigquery_check_operator.BigQueryValueCheckOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.bigquery.BigQueryGetDataOperator",
+ "airflow.contrib.operators.bigquery_get_data.BigQueryGetDataOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.bigquery.BigQueryExecuteQueryOperator",
+ "airflow.contrib.operators.bigquery_operator.BigQueryOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.bigquery.BigQueryDeleteTableOperator",
+ "airflow.contrib.operators.bigquery_table_delete_operator.BigQueryTableDeleteOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.gcs.GCSBucketCreateAclEntryOperator",
+ "airflow.contrib.operators.gcs_acl_operator.GoogleCloudStorageBucketCreateAclEntryOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.gcs.GCSObjectCreateAclEntryOperator",
+ "airflow.contrib.operators.gcs_acl_operator.GoogleCloudStorageObjectCreateAclEntryOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.gcs.GCSDeleteObjectsOperator",
+ "airflow.contrib.operators.gcs_delete_operator.GoogleCloudStorageDeleteOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.gcs.GCSListObjectsOperator",
+ "airflow.contrib.operators.gcs_list_operator.GoogleCloudStorageListOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.gcs.GCSCreateBucketOperator",
+ "airflow.contrib.operators.gcs_operator.GoogleCloudStorageCreateBucketOperator",
+ ),
+ (
+ "airflow.providers.amazon.aws.operators.athena.AWSAthenaOperator",
+ "airflow.contrib.operators.aws_athena_operator.AWSAthenaOperator",
+ ),
+ (
+ "airflow.providers.amazon.aws.operators.batch.AwsBatchOperator",
+ "airflow.contrib.operators.awsbatch_operator.AWSBatchOperator",
+ ),
+ (
+ "airflow.providers.amazon.aws.operators.sqs.SQSPublishOperator",
+ "airflow.contrib.operators.aws_sqs_publish_operator.SQSPublishOperator",
+ ),
+ (
+ "airflow.providers.amazon.aws.operators.sns.SnsPublishOperator",
+ "airflow.contrib.operators.sns_publish_operator.SnsPublishOperator",
+ ),
+ (
+ "airflow.providers.apache.druid.operators.druid.DruidOperator",
+ "airflow.contrib.operators.druid_operator.DruidOperator",
+ ),
+ (
+ "airflow.providers.apache.spark.operators.spark_jdbc.SparkSubmitOperator",
+ "airflow.contrib.operators.spark_jdbc_operator.SparkSubmitOperator",
+ ),
+ (
+ "airflow.providers.apache.spark.operators.spark_sql.SparkSqlOperator",
+ "airflow.contrib.operators.spark_sql_operator.SparkSqlOperator",
+ ),
+ (
+ "airflow.providers.apache.spark.operators.spark_submit.SparkSubmitOperator",
+ "airflow.contrib.operators.spark_submit_operator.SparkSubmitOperator",
+ ),
+ (
+ "airflow.providers.apache.spark.operators.spark_jdbc.SparkJDBCOperator",
+ "airflow.contrib.operators.spark_jdbc_operator.SparkJDBCOperator",
+ ),
+ (
+ "airflow.providers.apache.sqoop.operators.sqoop.SqoopOperator",
+ "airflow.contrib.operators.sqoop_operator.SqoopOperator",
+ ),
+ (
+ "airflow.providers.apache.druid.operators.druid_check.DruidCheckOperator",
+ "airflow.operators.druid_check_operator.DruidCheckOperator",
+ ),
+ (
+ "airflow.providers.apache.hive.operators.hive.HiveOperator",
+ "airflow.operators.hive_operator.HiveOperator",
+ ),
+ (
+ "airflow.providers.apache.hive.operators.hive_stats.HiveStatsCollectionOperator",
+ "airflow.operators.hive_stats_operator.HiveStatsCollectionOperator",
+ ),
+ (
+ "airflow.providers.apache.pig.operators.pig.PigOperator",
+ "airflow.operators.pig_operator.PigOperator",
+ ),
+ (
+ "airflow.providers.microsoft.azure.operators.adls_list.AzureDataLakeStorageListOperator",
+ "airflow.contrib.operators.adls_list_operator.AzureDataLakeStorageListOperator",
+ ),
+ (
+ "airflow.providers.microsoft.azure.operators"
+ ".azure_container_instances.AzureContainerInstancesOperator",
+ "airflow.contrib.operators.azure_container_instances_operator.AzureContainerInstancesOperator",
+ ),
+ (
+ "airflow.providers.microsoft.azure.operators.azure_cosmos.AzureCosmosInsertDocumentOperator",
+ "airflow.contrib.operators.azure_cosmos_operator.AzureCosmosInsertDocumentOperator",
+ ),
+ (
+ "airflow.providers.microsoft.azure.operators.wasb_delete_blob.WasbDeleteBlobOperator",
+ "airflow.contrib.operators.wasb_delete_blob_operator.WasbDeleteBlobOperator",
+ ),
+ (
+ "airflow.providers.amazon.aws.operators.ecs.ECSOperator",
+ "airflow.contrib.operators.ecs_operator.ECSOperator",
+ ),
+ (
+ "airflow.providers.amazon.aws.operators.emr_add_steps.EmrAddStepsOperator",
+ "airflow.contrib.operators.emr_add_steps_operator.EmrAddStepsOperator",
+ ),
+ (
+ "airflow.providers.amazon.aws.operators.emr_create_job_flow.EmrCreateJobFlowOperator",
+ "airflow.contrib.operators.emr_create_job_flow_operator.EmrCreateJobFlowOperator",
+ ),
+ (
+ "airflow.providers.amazon.aws.operators.emr_terminate_job_flow.EmrTerminateJobFlowOperator",
+ "airflow.contrib.operators.emr_terminate_job_flow_operator.EmrTerminateJobFlowOperator",
+ ),
+ (
+ "airflow.providers.amazon.aws.operators.s3_copy_object.S3CopyObjectOperator",
+ "airflow.contrib.operators.s3_copy_object_operator.S3CopyObjectOperator",
+ ),
+ (
+ "airflow.providers.amazon.aws.operators.s3_delete_objects.S3DeleteObjectsOperator",
+ "airflow.contrib.operators.s3_delete_objects_operator.S3DeleteObjectsOperator",
+ ),
+ (
+ "airflow.providers.amazon.aws.operators.s3_list.S3ListOperator",
+ "airflow.contrib.operators.s3_list_operator.S3ListOperator",
+ ),
+ (
+ "airflow.providers.amazon.aws.operators.sagemaker_base.SageMakerBaseOperator",
+ "airflow.contrib.operators.sagemaker_base_operator.SageMakerBaseOperator",
+ ),
+ (
+ "airflow.providers.amazon.aws.operators.sagemaker_endpoint_config.SageMakerEndpointConfigOperator",
+ "airflow.contrib.operators.sagemaker_endpoint_config_operator.SageMakerEndpointConfigOperator",
+ ),
+ (
+ "airflow.providers.amazon.aws.operators.sagemaker_endpoint.SageMakerEndpointOperator",
+ "airflow.contrib.operators.sagemaker_endpoint_operator.SageMakerEndpointOperator",
+ ),
+ (
+ "airflow.providers.amazon.aws.operators.sagemaker_model.SageMakerModelOperator",
+ "airflow.contrib.operators.sagemaker_model_operator.SageMakerModelOperator",
+ ),
+ (
+ "airflow.providers.amazon.aws.operators.sagemaker_training.SageMakerTrainingOperator",
+ "airflow.contrib.operators.sagemaker_training_operator.SageMakerTrainingOperator",
+ ),
+ (
+ "airflow.providers.amazon.aws.operators.sagemaker_transform.SageMakerTransformOperator",
+ "airflow.contrib.operators.sagemaker_transform_operator.SageMakerTransformOperator",
+ ),
+ (
+ "airflow.providers.amazon.aws.operators.sagemaker_tuning.SageMakerTuningOperator",
+ "airflow.contrib.operators.sagemaker_tuning_operator.SageMakerTuningOperator",
+ ),
+ (
+ "airflow.providers.docker.operators.docker_swarm.DockerSwarmOperator",
+ "airflow.contrib.operators.docker_swarm_operator.DockerSwarmOperator",
+ ),
+ (
+ "airflow.providers.cncf.kubernetes.operators.kubernetes_pod.KubernetesPodOperator",
+ "airflow.contrib.operators.kubernetes_pod_operator.KubernetesPodOperator",
+ ),
+ (
+ "airflow.providers.redis.operators.redis_publish.RedisPublishOperator",
+ "airflow.contrib.operators.redis_publish_operator.RedisPublishOperator",
+ ),
+ (
+ "airflow.operators.bash.BashOperator",
+ "airflow.operators.bash_operator.BashOperator",
+ ),
+ (
+ "airflow.providers.docker.operators.docker.DockerOperator",
+ "airflow.operators.docker_operator.DockerOperator",
+ ),
+ (
+ "airflow.providers.microsoft.mssql.operators.mssql.MsSqlOperator",
+ "airflow.operators.mssql_operator.MsSqlOperator",
+ ),
+ (
+ "airflow.providers.mysql.operators.mysql.MySqlOperator",
+ "airflow.operators.mysql_operator.MySqlOperator",
+ ),
+ (
+ "airflow.providers.oracle.operators.oracle.OracleOperator",
+ "airflow.operators.oracle_operator.OracleOperator",
+ ),
+ (
+ "airflow.providers.papermill.operators.papermill.PapermillOperator",
+ "airflow.operators.papermill_operator.PapermillOperator",
+ ),
+ (
+ "airflow.operators.sql.SQLCheckOperator",
+ "airflow.operators.presto_check_operator.PrestoCheckOperator",
+ ),
+ (
+ "airflow.operators.sql.SQLIntervalCheckOperator",
+ "airflow.operators.presto_check_operator.PrestoIntervalCheckOperator",
+ ),
+ (
+ "airflow.operators.sql.SQLValueCheckOperator",
+ "airflow.operators.presto_check_operator.PrestoValueCheckOperator",
+ ),
+ (
+ "airflow.operators.sql.SQLCheckOperator",
+ "airflow.operators.check_operator.CheckOperator",
+ ),
+ (
+ "airflow.operators.sql.SQLIntervalCheckOperator",
+ "airflow.operators.check_operator.IntervalCheckOperator",
+ ),
+ (
+ "airflow.operators.sql.SQLValueCheckOperator",
+ "airflow.operators.check_operator.ValueCheckOperator",
+ ),
+ (
+ "airflow.operators.sql.SQLThresholdCheckOperator",
+ "airflow.operators.check_operator.ThresholdCheckOperator",
+ ),
+ (
+ "airflow.operators.sql.BranchSQLOperator",
+ "airflow.operators.sql_branch_operator.BranchSqlOperator",
+ ),
+ (
+ "airflow.operators.python.BranchPythonOperator",
+ "airflow.operators.python_operator.BranchPythonOperator",
+ ),
+ (
+ "airflow.operators.python.PythonOperator",
+ "airflow.operators.python_operator.PythonOperator",
+ ),
+ (
+ "airflow.operators.python.ShortCircuitOperator",
+ "airflow.operators.python_operator.ShortCircuitOperator",
+ ),
+ (
+ "airflow.operators.python.PythonVirtualenvOperator",
+ "airflow.operators.python_operator.PythonVirtualenvOperator",
+ ),
+ (
+ "airflow.providers.sqlite.operators.sqlite.SqliteOperator",
+ "airflow.operators.sqlite_operator.SqliteOperator",
+ ),
+ (
+ "airflow.providers.databricks.operators.databricks.DatabricksRunNowOperator",
+ "airflow.contrib.operators.databricks_operator.DatabricksRunNowOperator",
+ ),
+ (
+ "airflow.providers.databricks.operators.databricks.DatabricksSubmitRunOperator",
+ "airflow.contrib.operators.databricks_operator.DatabricksSubmitRunOperator",
+ ),
+ (
+ "airflow.providers.dingding.operators.dingding.DingdingOperator",
+ "airflow.contrib.operators.dingding_operator.DingdingOperator",
+ ),
+ (
+ "airflow.providers.discord.operators.discord_webhook.DiscordWebhookOperator",
+ "airflow.contrib.operators.discord_webhook_operator.DiscordWebhookOperator",
+ ),
+ (
+ "airflow.providers.jenkins.operators.jenkins_job_trigger.JenkinsJobTriggerOperator",
+ "airflow.contrib.operators.jenkins_job_trigger_operator.JenkinsJobTriggerOperator",
+ ),
+ (
+ "airflow.providers.opsgenie.operators.opsgenie_alert.OpsgenieAlertOperator",
+ "airflow.contrib.operators.opsgenie_alert_operator.OpsgenieAlertOperator",
+ ),
+ (
+ "airflow.providers.qubole.operators.qubole_check.QuboleCheckOperator",
+ "airflow.contrib.operators.qubole_check_operator.QuboleCheckOperator",
+ ),
+ (
+ "airflow.providers.qubole.operators.qubole_check.QuboleValueCheckOperator",
+ "airflow.contrib.operators.qubole_check_operator.QuboleValueCheckOperator",
+ ),
+ (
+ "airflow.providers.qubole.operators.qubole.QuboleOperator",
+ "airflow.contrib.operators.qubole_operator.QuboleOperator",
+ ),
+ (
+ "airflow.providers.segment.operators.segment_track_event.SegmentTrackEventOperator",
+ "airflow.contrib.operators.segment_track_event_operator.SegmentTrackEventOperator",
+ ),
+ (
+ "airflow.providers.slack.operators.slack_webhook.SlackWebhookOperator",
+ "airflow.contrib.operators.slack_webhook_operator.SlackWebhookOperator",
+ ),
+ (
+ "airflow.providers.vertica.operators.vertica.VerticaOperator",
+ "airflow.contrib.operators.vertica_operator.VerticaOperator",
+ ),
+ (
+ "airflow.providers.slack.operators.slack.SlackAPIPostOperator",
+ "airflow.operators.slack_operator.SlackAPIPostOperator",
+ ),
+ (
+ "airflow.providers.slack.operators.slack.SlackAPIOperator",
+ "airflow.operators.slack_operator.SlackAPIOperator",
+ ),
+ (
+ "airflow.providers.grpc.operators.grpc.GrpcOperator",
+ "airflow.contrib.operators.grpc_operator.GrpcOperator",
+ ),
+ (
+ "airflow.providers.ssh.operators.ssh.SSHOperator",
+ "airflow.contrib.operators.ssh_operator.SSHOperator",
+ ),
+ (
+ "airflow.providers.microsoft.winrm.operators.winrm.WinRMOperator",
+ "airflow.contrib.operators.winrm_operator.WinRMOperator",
+ ),
+ (
+ "airflow.operators.email.EmailOperator",
+ "airflow.operators.email_operator.EmailOperator",
+ ),
+ (
+ "airflow.providers.http.operators.http.SimpleHttpOperator",
+ "airflow.operators.http_operator.SimpleHttpOperator",
+ ),
+ (
+ "airflow.providers.jdbc.operators.jdbc.JdbcOperator",
+ "airflow.operators.jdbc_operator.JdbcOperator",
+ ),
+ (
+ "airflow.providers.sftp.operators.sftp.SFTPOperator",
+ "airflow.contrib.operators.sftp_operator.SFTPOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.cloud_sql.CloudSQLBaseOperator",
+ "airflow.contrib.operators.gcp_sql_operator.CloudSqlBaseOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.cloud_sql.CloudSQLCreateInstanceDatabaseOperator",
+ "airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabaseCreateOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.cloud_sql.CloudSQLCreateInstanceOperator",
+ "airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceCreateOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.cloud_sql.CloudSQLDeleteInstanceDatabaseOperator",
+ "airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabaseDeleteOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.cloud_sql.CloudSQLDeleteInstanceOperator",
+ "airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDeleteOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.cloud_sql.CloudSQLExecuteQueryOperator",
+ "airflow.contrib.operators.gcp_sql_operator.CloudSqlQueryOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.cloud_sql.CloudSQLExportInstanceOperator",
+ "airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceExportOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.cloud_sql.CloudSQLImportInstanceOperator",
+ "airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceImportOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.cloud_sql.CloudSQLInstancePatchOperator",
+ "airflow.contrib.operators.gcp_sql_operator.CloudSqlInstancePatchOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.cloud_sql.CloudSQLPatchInstanceDatabaseOperator",
+ "airflow.contrib.operators.gcp_sql_operator.CloudSqlInstanceDatabasePatchOperator",
+ ),
+ (
+ "airflow.providers.jira.operators.jira.JiraOperator",
+ "airflow.contrib.operators.jira_operator.JiraOperator",
+ ),
+ (
+ "airflow.providers.postgres.operators.postgres.PostgresOperator",
+ "airflow.operators.postgres_operator.PostgresOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.speech_to_text.CloudSpeechToTextRecognizeSpeechOperator",
+ "airflow.contrib.operators.gcp_speech_to_text_operator.GcpSpeechToTextRecognizeSpeechOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.text_to_speech.CloudTextToSpeechSynthesizeOperator",
+ "airflow.contrib.operators.gcp_text_to_speech_operator.GcpTextToSpeechSynthesizeOperator",
+ ),
+]
+
+SECRETS = [
+ (
+ "airflow.providers.amazon.aws.secrets.secrets_manager.SecretsManagerBackend",
+ "airflow.contrib.secrets.aws_secrets_manager.SecretsManagerBackend",
+ ),
+ (
+ "airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
+ "airflow.contrib.secrets.aws_systems_manager.SystemsManagerParameterStoreBackend",
+ ),
+ (
+ "airflow.providers.google.cloud.secrets.secret_manager.CloudSecretManagerBackend",
+ "airflow.contrib.secrets.gcp_secrets_manager.CloudSecretsManagerBackend",
+ ),
+ (
+ "airflow.providers.hashicorp.secrets.vault.VaultBackend",
+ "airflow.contrib.secrets.hashicorp_vault.VaultBackend",
+ ),
+]
+
+SENSORS = [
+ (
+ "airflow.providers.apache.cassandra.sensors.record.CassandraRecordSensor",
+ "airflow.contrib.sensors.cassandra_record_sensor.CassandraRecordSensor",
+ ),
+ (
+ "airflow.providers.apache.cassandra.sensors.table.CassandraTableSensor",
+ "airflow.contrib.sensors.cassandra_table_sensor.CassandraTableSensor",
+ ),
+ (
+ "airflow.providers.datadog.sensors.datadog.DatadogSensor",
+ "airflow.contrib.sensors.datadog_sensor.DatadogSensor",
+ ),
+ (
+ "airflow.providers.google.cloud.sensors.bigtable.BigtableTableReplicationCompletedSensor",
+ "airflow.contrib.operators.gcp_bigtable_operator."
+ "BigtableTableWaitForReplicationSensor",
+ ),
+ (
+ "airflow.providers.google.cloud.sensors.cloud_storage_transfer_service."
+ "CloudDataTransferServiceJobStatusSensor",
+ "airflow.contrib.sensors.gcp_transfer_sensor."
+ "GCPTransferServiceWaitForJobStatusSensor",
+ ),
+ (
+ "airflow.providers.google.cloud.sensors.pubsub.PubSubPullSensor",
+ "airflow.contrib.sensors.pubsub_sensor.PubSubPullSensor",
+ ),
+ (
+ "airflow.providers.google.cloud.sensors.bigquery.BigQueryTableExistenceSensor",
+ "airflow.contrib.sensors.bigquery_sensor.BigQueryTableSensor",
+ ),
+ (
+ "airflow.providers.google.cloud.sensors.gcs.GCSObjectExistenceSensor",
+ "airflow.contrib.sensors.gcs_sensor.GoogleCloudStorageObjectSensor",
+ ),
+ (
+ "airflow.providers.google.cloud.sensors.gcs.GCSObjectUpdateSensor",
+ "airflow.contrib.sensors.gcs_sensor.GoogleCloudStorageObjectUpdatedSensor",
+ ),
+ (
+ "airflow.providers.google.cloud.sensors.gcs.GCSObjectsWtihPrefixExistenceSensor",
+ "airflow.contrib.sensors.gcs_sensor.GoogleCloudStoragePrefixSensor",
+ ),
+ (
+ "airflow.providers.google.cloud.sensors.gcs.GCSUploadSessionCompleteSensor",
+ "airflow.contrib.sensors.gcs_sensor.GoogleCloudStorageUploadSessionCompleteSensor",
+ ),
+ (
+ "airflow.providers.amazon.aws.sensors.athena.AthenaSensor",
+ "airflow.contrib.sensors.aws_athena_sensor.AthenaSensor",
+ ),
+ (
+ "airflow.providers.amazon.aws.sensors.sqs.SQSSensor",
+ "airflow.contrib.sensors.aws_sqs_sensor.SQSSensor",
+ ),
+ (
+ "airflow.providers.apache.hdfs.sensors.hdfs.HdfsFolderSensor",
+ "airflow.contrib.sensors.hdfs_sensor.HdfsSensorFolder",
+ ),
+ (
+ "airflow.providers.apache.hdfs.sensors.hdfs.HdfsRegexSensor",
+ "airflow.contrib.sensors.hdfs_sensor.HdfsSensorRegex",
+ ),
+ (
+ "airflow.providers.apache.hive.sensors.hive_partition.HivePartitionSensor",
+ "airflow.sensors.hive_partition_sensor.HivePartitionSensor",
+ ),
+ (
+ "airflow.providers.apache.hive.sensors.metastore_partition.MetastorePartitionSensor",
+ "airflow.sensors.metastore_partition_sensor.MetastorePartitionSensor",
+ ),
+ (
+ "airflow.providers.apache.hive.sensors.named_hive_partition.NamedHivePartitionSensor",
+ "airflow.sensors.named_hive_partition_sensor.NamedHivePartitionSensor",
+ ),
+ (
+ "airflow.providers.apache.hdfs.sensors.web_hdfs.WebHdfsSensor",
+ "airflow.sensors.web_hdfs_sensor.WebHdfsSensor",
+ ),
+ (
+ "airflow.providers.apache.hdfs.sensors.hdfs.HdfsSensor",
+ "airflow.sensors.hdfs_sensor.HdfsSensor",
+ ),
+ (
+ "airflow.sensors.weekday_sensor.DayOfWeekSensor",
+ "airflow.contrib.sensors.weekday_sensor.DayOfWeekSensor",
+ ),
+ (
+ "airflow.sensors.filesystem.FileSensor",
+ "airflow.contrib.sensors.file_sensor.FileSensor",
+ ),
+ (
+ "airflow.providers.microsoft.azure.sensors.wasb.WasbBlobSensor",
+ "airflow.contrib.sensors.wasb_sensor.WasbBlobSensor",
+ ),
+ (
+ "airflow.providers.microsoft.azure.sensors.wasb.WasbPrefixSensor",
+ "airflow.contrib.sensors.wasb_sensor.WasbPrefixSensor",
+ ),
+ (
+ "airflow.providers.amazon.aws.sensors.glue_catalog_partition.AwsGlueCatalogPartitionSensor",
+ "airflow.contrib.sensors.aws_glue_catalog_partition_sensor.AwsGlueCatalogPartitionSensor",
+ ),
+ (
+ "airflow.providers.amazon.aws.sensors.emr_base.EmrBaseSensor",
+ "airflow.contrib.sensors.emr_base_sensor.EmrBaseSensor",
+ ),
+ (
+ "airflow.providers.amazon.aws.sensors.emr_job_flow.EmrJobFlowSensor",
+ "airflow.contrib.sensors.emr_job_flow_sensor.EmrJobFlowSensor",
+ ),
+ (
+ "airflow.providers.amazon.aws.sensors.emr_step.EmrStepSensor",
+ "airflow.contrib.sensors.emr_step_sensor.EmrStepSensor",
+ ),
+ (
+ "airflow.providers.amazon.aws.sensors.sagemaker_base.SageMakerBaseSensor",
+ "airflow.contrib.sensors.sagemaker_base_sensor.SageMakerBaseSensor",
+ ),
+ (
+ "airflow.providers.amazon.aws.sensors.sagemaker_endpoint.SageMakerEndpointSensor",
+ "airflow.contrib.sensors.sagemaker_endpoint_sensor.SageMakerEndpointSensor",
+ ),
+ (
+ "airflow.providers.amazon.aws.sensors.sagemaker_transform.SageMakerTransformSensor",
+ "airflow.contrib.sensors.sagemaker_transform_sensor.SageMakerTransformSensor",
+ ),
+ (
+ "airflow.providers.amazon.aws.sensors.sagemaker_tuning.SageMakerTuningSensor",
+ "airflow.contrib.sensors.sagemaker_tuning_sensor.SageMakerTuningSensor",
+ ),
+ (
+ "airflow.providers.amazon.aws.operators.s3_file_transform.S3FileTransformOperator",
+ "airflow.operators.s3_file_transform_operator.S3FileTransformOperator",
+ ),
+ (
+ "airflow.providers.amazon.aws.sensors.s3_key.S3KeySensor",
+ "airflow.sensors.s3_key_sensor.S3KeySensor",
+ ),
+ (
+ "airflow.providers.amazon.aws.sensors.s3_prefix.S3PrefixSensor",
+ "airflow.sensors.s3_prefix_sensor.S3PrefixSensor",
+ ),
+ (
+ "airflow.sensors.bash.BashSensor",
+ "airflow.contrib.sensors.bash_sensor.BashSensor",
+ ),
+ (
+ "airflow.providers.celery.sensors.celery_queue.CeleryQueueSensor",
+ "airflow.contrib.sensors.celery_queue_sensor.CeleryQueueSensor",
+ ),
+ (
+ "airflow.providers.mongo.sensors.mongo.MongoSensor",
+ "airflow.contrib.sensors.mongo_sensor.MongoSensor",
+ ),
+ (
+ "airflow.sensors.python.PythonSensor",
+ "airflow.contrib.sensors.python_sensor.PythonSensor",
+ ),
+ (
+ "airflow.providers.redis.sensors.redis_key.RedisKeySensor",
+ "airflow.contrib.sensors.redis_key_sensor.RedisKeySensor",
+ ),
+ (
+ "airflow.providers.redis.sensors.redis_pub_sub.RedisPubSubSensor",
+ "airflow.contrib.sensors.redis_pub_sub_sensor.RedisPubSubSensor",
+ ),
+ (
+ "airflow.providers.datadog.sensors.datadog.DatadogSensor",
+ "airflow.contrib.sensors.datadog_sensor.DatadogSensor",
+ ),
+ (
+ "airflow.providers.qubole.sensors.qubole.QuboleSensor",
+ "airflow.contrib.sensors.qubole_sensor.QuboleSensor",
+ ),
+ (
+ "airflow.providers.qubole.sensors.qubole.QubolePartitionSensor",
+ "airflow.contrib.sensors.qubole_sensor.QubolePartitionSensor",
+ ),
+ (
+ "airflow.providers.qubole.sensors.qubole.QuboleFileSensor",
+ "airflow.contrib.sensors.qubole_sensor.QuboleFileSensor",
+ ),
+ (
+ "airflow.providers.ftp.sensors.ftp.FTPSensor",
+ "airflow.contrib.sensors.ftp_sensor.FTPSensor",
+ ),
+ (
+ "airflow.providers.ftp.sensors.ftp.FTPSSensor",
+ "airflow.contrib.sensors.ftp_sensor.FTPSSensor",
+ ),
+ (
+ "airflow.providers.imap.sensors.imap_attachment.ImapAttachmentSensor",
+ "airflow.contrib.sensors.imap_attachment_sensor.ImapAttachmentSensor",
+ ),
+ (
+ "airflow.providers.jira.sensors.jira.JiraSensor",
+ "airflow.contrib.sensors.jira_sensor.JiraSensor",
+ ),
+ (
+ "airflow.providers.jira.sensors.jira.JiraTicketSensor",
+ "airflow.contrib.sensors.jira_sensor.JiraTicketSensor",
+ ),
+ (
+ "airflow.providers.http.sensors.http.HttpSensor",
+ "airflow.sensors.http_sensor.HttpSensor",
+ ),
+ (
+ "airflow.providers.sftp.sensors.sftp.SFTPSensor",
+ "airflow.contrib.sensors.sftp_sensor.SFTPSensor",
+ ),
+]
+
+TRANSFERS = [
+ (
+ "airflow.providers.google.cloud.transfers.local_to_gcs.LocalFilesystemToGCSOperator",
+ "airflow.contrib.operators.file_to_gcs.FileToGoogleCloudStorageOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.transfers.adls_to_gcs.ADLSToGCSOperator",
+ "airflow.contrib.operators.adls_to_gcs.AdlsToGoogleCloudStorageOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.cloud_storage_transfer_service."
+ "CloudDataTransferServiceGCSToGCSOperator",
+ "airflow.contrib.operators.gcp_transfer_operator."
+ "GoogleCloudStorageToGoogleCloudStorageTransferOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.vision.CloudVisionAddProductToProductSetOperator",
+ "airflow.contrib.operators.gcp_vision_operator.CloudVisionAddProductToProductSetOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.transfers.gcs_to_bigquery.GCSToBigQueryOperator",
+ "airflow.contrib.operators.gcs_to_bq.GoogleCloudStorageToBigQueryOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.transfers.gcs_to_gcs.GCSToGCSOperator",
+ "airflow.contrib.operators.gcs_to_gcs.GoogleCloudStorageToGoogleCloudStorageOperator",
+ ),
+ (
+ "airflow.providers.amazon.aws.transfers.gcs_to_s3.GCSToS3Operator",
+ "airflow.contrib.operators.gcs_to_s3.GoogleCloudStorageToS3Operator",
+ ),
+ (
+ "airflow.providers.google.cloud.transfers.mssql_to_gcs.MSSQLToGCSOperator",
+ "airflow.contrib.operators.mssql_to_gcs.MsSqlToGoogleCloudStorageOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.transfers.mysql_to_gcs.MySQLToGCSOperator",
+ "airflow.contrib.operators.mysql_to_gcs.MySqlToGoogleCloudStorageOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.transfers.postgres_to_gcs.PostgresToGCSOperator",
+ "airflow.contrib.operators.postgres_to_gcs_operator."
+ "PostgresToGoogleCloudStorageOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.transfers.bigquery_to_bigquery.BigQueryToBigQueryOperator",
+ "airflow.contrib.operators.bigquery_to_bigquery.BigQueryToBigQueryOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.transfers.bigquery_to_gcs.BigQueryToGCSOperator",
+ "airflow.contrib.operators.bigquery_to_gcs.BigQueryToCloudStorageOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.transfers.bigquery_to_mysql.BigQueryToMySqlOperator",
+ "airflow.contrib.operators.bigquery_to_mysql_operator.BigQueryToMySqlOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.transfers.sql_to_gcs.BaseSQLToGCSOperator",
+ "airflow.contrib.operators.sql_to_gcs.BaseSQLToGoogleCloudStorageOperator",
+ ),
+ (
+ "airflow.providers.amazon.aws.transfers.dynamodb_to_s3.DynamoDBToS3Operator",
+ "airflow.contrib.operators.dynamodb_to_s3.DynamoDBToS3Operator",
+ ),
+ (
+ "airflow.providers.amazon.aws.transfers.hive_to_dynamodb.HiveToDynamoDBOperator",
+ "airflow.contrib.operators.hive_to_dynamodb.HiveToDynamoDBOperator",
+ ),
+ (
+ "airflow.providers.amazon.aws.transfers.imap_attachment_to_s3.ImapAttachmentToS3Operator",
+ "airflow.contrib.operators.imap_attachment_to_s3_operator.ImapAttachmentToS3Operator",
+ ),
+ (
+ "airflow.providers.amazon.aws.transfers.mongo_to_s3.MongoToS3Operator",
+ "airflow.contrib.operators.mongo_to_s3.MongoToS3Operator",
+ ),
+ (
+ "airflow.providers.amazon.aws.transfers.s3_to_sftp.S3ToSFTPOperator",
+ "airflow.contrib.operators.s3_to_sftp_operator.S3ToSFTPOperator",
+ ),
+ (
+ "airflow.providers.amazon.aws.transfers.sftp_to_s3.SFTPToS3Operator",
+ "airflow.contrib.operators.sftp_to_s3_operator.SFTPToS3Operator",
+ ),
+ (
+ "airflow.providers.amazon.aws.transfers.gcs_to_s3.GCSToS3Operator",
+ "airflow.operators.gcs_to_s3.GCSToS3Operator",
+ ),
+ (
+ "airflow.providers.amazon.aws.transfers.google_api_to_s3.GoogleApiToS3Operator",
+ "airflow.operators.google_api_to_s3_transfer.GoogleApiToS3Transfer",
+ ),
+ (
+ "airflow.providers.amazon.aws.transfers.redshift_to_s3.RedshiftToS3Operator",
+ "airflow.operators.redshift_to_s3_operator.RedshiftToS3Transfer",
+ ),
+ (
+ "airflow.providers.amazon.aws.transfers.s3_to_redshift.S3ToRedshiftOperator",
+ "airflow.operators.s3_to_redshift_operator.S3ToRedshiftTransfer",
+ ),
+ (
+ "airflow.providers.apache.hive.transfers.vertica_to_hive.VerticaToHiveOperator",
+ "airflow.contrib.operators.vertica_to_hive.VerticaToHiveTransfer",
+ ),
+ (
+ "airflow.providers.apache.druid.transfers.hive_to_druid.HiveToDruidOperator",
+ "airflow.operators.hive_to_druid.HiveToDruidTransfer",
+ ),
+ (
+ "airflow.providers.apache.hive.transfers.hive_to_mysql.HiveToMySqlOperator",
+ "airflow.operators.hive_to_mysql.HiveToMySqlTransfer",
+ ),
+ (
+ "airflow.providers.apache.hive.transfers.mysql_to_hive.MySqlToHiveOperator",
+ "airflow.operators.mysql_to_hive.MySqlToHiveTransfer",
+ ),
+ (
+ "airflow.providers.apache.hive.transfers.s3_to_hive.S3ToHiveOperator",
+ "airflow.operators.s3_to_hive_operator.S3ToHiveTransfer",
+ ),
+ (
+ "airflow.providers.apache.hive.transfers.hive_to_samba.HiveToSambaOperator",
+ "airflow.operators.hive_to_samba_operator.HiveToSambaOperator",
+ ),
+ (
+ "airflow.providers.apache.hive.transfers.mssql_to_hive.MsSqlToHiveOperator",
+ "airflow.operators.mssql_to_hive.MsSqlToHiveTransfer",
+ ),
+ (
+ "airflow.providers.microsoft.azure.transfers.file_to_wasb.FileToWasbOperator",
+ "airflow.contrib.operators.file_to_wasb.FileToWasbOperator",
+ ),
+ (
+ "airflow.providers.google.suite.transfers.gcs_to_gdrive.GCSToGoogleDriveOperator",
+ "airflow.contrib.operators.gcs_to_gdrive_operator.GCSToGoogleDriveOperator",
+ ),
+ (
+ "airflow.providers.microsoft.azure.transfers.oracle_to_azure_data_lake"
+ ".OracleToAzureDataLakeOperator",
+ "airflow.contrib.operators.oracle_to_azure_data_lake_transfer.OracleToAzureDataLakeOperator",
+ ),
+ (
+ "airflow.providers.oracle.transfers.oracle_to_oracle.OracleToOracleOperator",
+ "airflow.contrib.operators.oracle_to_oracle_transfer.OracleToOracleTransfer",
+ ),
+ (
+ "airflow.providers.google.cloud.transfers.s3_to_gcs.S3ToGCSOperator",
+ "airflow.contrib.operators.s3_to_gcs_operator.S3ToGCSOperator",
+ ),
+ (
+ "airflow.providers.mysql.transfers.vertica_to_mysql.VerticaToMySqlOperator",
+ "airflow.contrib.operators.vertica_to_mysql.VerticaToMySqlTransfer",
+ ),
+ (
+ "airflow.providers.mysql.transfers.presto_to_mysql.PrestoToMySqlOperator",
+ "airflow.operators.presto_to_mysql.PrestoToMySqlTransfer",
+ ),
+ (
+ "airflow.providers.google.cloud.operators.cloud_storage_transfer_service"
+ ".CloudDataTransferServiceS3ToGCSOperator",
+ "airflow.contrib.operators.s3_to_gcs_transfer_operator.CloudDataTransferServiceS3ToGCSOperator",
+ ),
+ (
+ "airflow.providers.google.cloud.transfers.cassandra_to_gcs.CassandraToGCSOperator",
+ "airflow.contrib.operators.cassandra_to_gcs.CassandraToGoogleCloudStorageOperator",
+ ),
+]
+
+ALL = OPERATORS + HOOKS + SECRETS + SENSORS + TRANSFERS
diff --git a/tests/upgrade/rules/test_import_changes.py b/tests/upgrade/rules/test_import_changes.py
new file mode 100644
index 0000000..8ae3a8c
--- /dev/null
+++ b/tests/upgrade/rules/test_import_changes.py
@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from tempfile import NamedTemporaryFile
+from tests.compat import mock
+
+from airflow.upgrade.rules.import_changes import ImportChange, ImportChangesRule
+
+OLD_CLASS = "OldOperator"
+NEW_CLASS = "NewOperator"
+PROVIDER = "dummy"
+OLD_PATH = "airflow.contrib." + OLD_CLASS
+NEW_PATH = "airflow.providers." + PROVIDER + "." + NEW_CLASS
+
+
+class TestImportChange:
+ def test_info(self):
+ change = ImportChange(
+ old_path=OLD_PATH, new_path=NEW_PATH, providers_package=PROVIDER
+ )
+ assert change.info(
+ "file.py"
+ ) == "Using `{}` will be replaced by `{}` and requires `{}` providers package. " \
+ "Affected file: file.py".format(OLD_PATH, NEW_PATH, PROVIDER)
+ assert change.old_class == OLD_CLASS
+ assert change.new_class == NEW_CLASS
+
+ def test_from_new_old_paths(self):
+ paths_tuple = (NEW_PATH, OLD_PATH)
+ change = ImportChange.from_new_old_paths(*paths_tuple)
+ assert change.info() == "Using `{}` will be replaced by `{}` and requires `{}` " \
+ "providers package".format(OLD_PATH, NEW_PATH, PROVIDER)
+
+
+class TestImportChangesRule:
+ @mock.patch("airflow.upgrade.rules.import_changes.list_py_file_paths")
+ @mock.patch(
+ "airflow.upgrade.rules.import_changes.ImportChangesRule.ALL_CHANGES",
+ [ImportChange.from_new_old_paths(NEW_PATH, OLD_PATH)],
+ )
+ def test_check(self, mock_list_files):
+ with NamedTemporaryFile("w+") as temp:
+ mock_list_files.return_value = [temp.name]
+
+ temp.write("from airflow.contrib import %s" % OLD_CLASS)
+ temp.flush()
+ msgs = ImportChangesRule().check()
+
+ assert len(msgs) == 1
+ msg = msgs[0]
+ assert temp.name in msg
+ assert OLD_PATH in msg
+ assert OLD_CLASS in msg
+ assert "requires `{}`".format(PROVIDER) in msg