You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by as...@apache.org on 2020/10/11 19:50:42 UTC
[airflow] branch master updated: Change prefix of AwsDynamoDB hook
module (#11209)
This is an automated email from the ASF dual-hosted git repository.
ash pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new c3e3405 Change prefix of AwsDynamoDB hook module (#11209)
c3e3405 is described below
commit c3e340584bf1892c4f73aa9e7495b5823dab0c40
Author: eladkal <45...@users.noreply.github.com>
AuthorDate: Sun Oct 11 22:49:23 2020 +0300
Change prefix of AwsDynamoDB hook module (#11209)
* align import path of AwsDynamoDBHook in aws providers
Co-authored-by: Tomek Urbaszek <tu...@gmail.com>
---
airflow/contrib/hooks/aws_dynamodb_hook.py | 6 +--
airflow/providers/amazon/aws/ADDITIONAL_INFO.md | 26 ++++++++++
airflow/providers/amazon/aws/hooks/aws_dynamodb.py | 56 ++++------------------
.../aws/hooks/{aws_dynamodb.py => dynamodb.py} | 0
.../amazon/aws/transfers/dynamodb_to_s3.py | 2 +-
.../amazon/aws/transfers/hive_to_dynamodb.py | 2 +-
docs/operators-and-hooks-ref.rst | 2 +-
tests/core/test_project_structure.py | 34 ++++++++++---
tests/deprecated_classes.py | 2 +-
.../{test_aws_dynamodb.py => test_dynamodb.py} | 2 +-
.../amazon/aws/transfers/test_hive_to_dynamodb.py | 2 +-
11 files changed, 72 insertions(+), 62 deletions(-)
diff --git a/airflow/contrib/hooks/aws_dynamodb_hook.py b/airflow/contrib/hooks/aws_dynamodb_hook.py
index 0656d8a..9ea6e1d 100644
--- a/airflow/contrib/hooks/aws_dynamodb_hook.py
+++ b/airflow/contrib/hooks/aws_dynamodb_hook.py
@@ -15,14 +15,14 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-"""This module is deprecated. Please use `airflow.providers.amazon.aws.hooks.aws_dynamodb`."""
+"""This module is deprecated. Please use `airflow.providers.amazon.aws.hooks.dynamodb`."""
import warnings
# pylint: disable=unused-import
-from airflow.providers.amazon.aws.hooks.aws_dynamodb import AwsDynamoDBHook # noqa
+from airflow.providers.amazon.aws.hooks.dynamodb import AwsDynamoDBHook # noqa
warnings.warn(
- "This module is deprecated. Please use `airflow.providers.amazon.aws.hooks.aws_dynamodb`.",
+ "This module is deprecated. Please use `airflow.providers.amazon.aws.hooks.dynamodb`.",
DeprecationWarning, stacklevel=2
)
diff --git a/airflow/providers/amazon/aws/ADDITIONAL_INFO.md b/airflow/providers/amazon/aws/ADDITIONAL_INFO.md
new file mode 100644
index 0000000..f0b0464
--- /dev/null
+++ b/airflow/providers/amazon/aws/ADDITIONAL_INFO.md
@@ -0,0 +1,26 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ -->
+
+## Change in import paths
+
+If you are upgrading from 2020.10.5 note the following changes in import paths
+
+| Old path | New path |
+| --------------------------------------------------------------- | ----------------------------------------------------------- |
+| airflow.providers.amazon.aws.hooks.aws_dynamodb.AwsDynamoDBHook | airflow.providers.amazon.aws.hooks.dynamodb.AwsDynamoDBHook |
diff --git a/airflow/providers/amazon/aws/hooks/aws_dynamodb.py b/airflow/providers/amazon/aws/hooks/aws_dynamodb.py
index 4750436..fc6bd1b 100644
--- a/airflow/providers/amazon/aws/hooks/aws_dynamodb.py
+++ b/airflow/providers/amazon/aws/hooks/aws_dynamodb.py
@@ -15,53 +15,15 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+"""This module is deprecated. Please use `airflow.providers.amazon.aws.hooks.dynamodb`."""
+import warnings
-"""
-This module contains the AWS DynamoDB hook
-"""
-from typing import Iterable, List, Optional
+# pylint: disable=unused-import
+from airflow.providers.amazon.aws.hooks.dynamodb import AwsDynamoDBHook # noqa
-from airflow.exceptions import AirflowException
-from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
-
-
-class AwsDynamoDBHook(AwsBaseHook):
- """
- Interact with AWS DynamoDB.
-
- Additional arguments (such as ``aws_conn_id``) may be specified and
- are passed down to the underlying AwsBaseHook.
-
- .. seealso::
- :class:`~airflow.providers.amazon.aws.hooks.base_aws.AwsBaseHook`
-
- :param table_keys: partition key and sort key
- :type table_keys: list
- :param table_name: target DynamoDB table
- :type table_name: str
- """
-
- def __init__(
- self, *args, table_keys: Optional[List] = None, table_name: Optional[str] = None, **kwargs
- ) -> None:
- self.table_keys = table_keys
- self.table_name = table_name
- kwargs["resource_type"] = "dynamodb"
- super().__init__(*args, **kwargs)
-
- def write_batch_data(self, items: Iterable) -> bool:
- """
- Write batch items to DynamoDB table with provisioned throughout capacity.
- """
- try:
- table = self.get_conn().Table(self.table_name)
-
- with table.batch_writer(overwrite_by_pkeys=self.table_keys) as batch:
- for item in items:
- batch.put_item(Item=item)
- return True
- except Exception as general_error:
- raise AirflowException(
- "Failed to insert items in dynamodb, error: {error}".format(error=str(general_error))
- )
+warnings.warn(
+ "This module is deprecated. Please use `airflow.providers.amazon.aws.hooks.dynamodb`.",
+ DeprecationWarning,
+ stacklevel=2,
+)
diff --git a/airflow/providers/amazon/aws/hooks/aws_dynamodb.py b/airflow/providers/amazon/aws/hooks/dynamodb.py
similarity index 100%
copy from airflow/providers/amazon/aws/hooks/aws_dynamodb.py
copy to airflow/providers/amazon/aws/hooks/dynamodb.py
diff --git a/airflow/providers/amazon/aws/transfers/dynamodb_to_s3.py b/airflow/providers/amazon/aws/transfers/dynamodb_to_s3.py
index 493b6ea..10cadd5 100644
--- a/airflow/providers/amazon/aws/transfers/dynamodb_to_s3.py
+++ b/airflow/providers/amazon/aws/transfers/dynamodb_to_s3.py
@@ -29,7 +29,7 @@ from typing import Any, Callable, Dict, Optional
from uuid import uuid4
from airflow.models import BaseOperator
-from airflow.providers.amazon.aws.hooks.aws_dynamodb import AwsDynamoDBHook
+from airflow.providers.amazon.aws.hooks.dynamodb import AwsDynamoDBHook
from airflow.providers.amazon.aws.hooks.s3 import S3Hook
from airflow.utils.decorators import apply_defaults
diff --git a/airflow/providers/amazon/aws/transfers/hive_to_dynamodb.py b/airflow/providers/amazon/aws/transfers/hive_to_dynamodb.py
index 0ca7218..574d73a 100644
--- a/airflow/providers/amazon/aws/transfers/hive_to_dynamodb.py
+++ b/airflow/providers/amazon/aws/transfers/hive_to_dynamodb.py
@@ -23,7 +23,7 @@ This module contains operator to move data from Hive to DynamoDB.
import json
from airflow.models import BaseOperator
-from airflow.providers.amazon.aws.hooks.aws_dynamodb import AwsDynamoDBHook
+from airflow.providers.amazon.aws.hooks.dynamodb import AwsDynamoDBHook
from airflow.providers.apache.hive.hooks.hive import HiveServer2Hook
from airflow.utils.decorators import apply_defaults
diff --git a/docs/operators-and-hooks-ref.rst b/docs/operators-and-hooks-ref.rst
index 421ebea..9b17642 100644
--- a/docs/operators-and-hooks-ref.rst
+++ b/docs/operators-and-hooks-ref.rst
@@ -477,7 +477,7 @@ These integrations allow you to perform various operations within the Amazon Web
* - `Amazon DynamoDB <https://aws.amazon.com/dynamodb/>`__
-
- - :mod:`airflow.providers.amazon.aws.hooks.aws_dynamodb`
+ - :mod:`airflow.providers.amazon.aws.hooks.dynamodb`
-
-
diff --git a/tests/core/test_project_structure.py b/tests/core/test_project_structure.py
index 9b1d967..21a90b6 100644
--- a/tests/core/test_project_structure.py
+++ b/tests/core/test_project_structure.py
@@ -56,18 +56,23 @@ class TestProjectStructure(unittest.TestCase):
"""
Assert every module in /airflow/providers has a corresponding test_ file in tests/airflow/providers.
"""
+ # Deprecated modules that don't have corresponded test
+ expected_missing_providers_modules = {('airflow/providers/amazon/aws/hooks/aws_dynamodb.py',
+ 'tests/providers/amazon/aws/hooks/test_aws_dynamodb.py')}
+
# TODO: Should we extend this test to cover other directories?
- expected_test_files = glob.glob(f"{ROOT_FOLDER}/airflow/providers/**/*.py", recursive=True)
+ modules_files = glob.glob(f"{ROOT_FOLDER}/airflow/providers/**/*.py", recursive=True)
+
# Make path relative
- expected_test_files = (os.path.relpath(f, ROOT_FOLDER) for f in expected_test_files)
+ modules_files = (os.path.relpath(f, ROOT_FOLDER) for f in modules_files)
# Exclude example_dags
- expected_test_files = (f for f in expected_test_files if "/example_dags/" not in f)
+ modules_files = (f for f in modules_files if "/example_dags/" not in f)
# Exclude __init__.py
- expected_test_files = (f for f in expected_test_files if not f.endswith("__init__.py"))
+ modules_files = (f for f in modules_files if not f.endswith("__init__.py"))
# Change airflow/ to tests/
expected_test_files = (
f'tests/{f.partition("/")[2]}'
- for f in expected_test_files if not f.endswith("__init__.py")
+ for f in modules_files if not f.endswith("__init__.py")
)
# Add test_ prefix to filename
expected_test_files = (
@@ -81,11 +86,28 @@ class TestProjectStructure(unittest.TestCase):
# Exclude __init__.py
current_test_files = (f for f in current_test_files if not f.endswith("__init__.py"))
+ modules_files = set(modules_files)
expected_test_files = set(expected_test_files)
current_test_files = set(current_test_files)
missing_tests_files = expected_test_files - expected_test_files.intersection(current_test_files)
- self.assertEqual(set(), missing_tests_files)
+
+ with self.subTest("Detect missing tests in providers module"):
+ expected_missing_test_modules = set(pair[1] for pair in expected_missing_providers_modules)
+ missing_tests_files = missing_tests_files - set(expected_missing_test_modules)
+ self.assertEqual(set(), missing_tests_files)
+
+ with self.subTest("Verify removed deprecated module also removed from deprecated list"):
+ expected_missing_modules = set(pair[0] for pair in expected_missing_providers_modules)
+ removed_deprecated_module = expected_missing_modules - modules_files
+ if removed_deprecated_module:
+ self.fail(
+ "You've removed a deprecated module:\n"
+ f"{removed_deprecated_module}"
+ "\n"
+ "Thank you very much.\n"
+ "Can you remove it from the list of expected missing modules tests, please?"
+ )
class TestGoogleProviderProjectStructure(unittest.TestCase):
diff --git a/tests/deprecated_classes.py b/tests/deprecated_classes.py
index 855239e..170b0c7 100644
--- a/tests/deprecated_classes.py
+++ b/tests/deprecated_classes.py
@@ -405,7 +405,7 @@ HOOKS = [
'airflow.contrib.hooks.aws_hook.AwsHook',
),
(
- 'airflow.providers.amazon.aws.hooks.aws_dynamodb.AwsDynamoDBHook',
+ 'airflow.providers.amazon.aws.hooks.dynamodb.AwsDynamoDBHook',
'airflow.contrib.hooks.aws_dynamodb_hook.AwsDynamoDBHook',
),
(
diff --git a/tests/providers/amazon/aws/hooks/test_aws_dynamodb.py b/tests/providers/amazon/aws/hooks/test_dynamodb.py
similarity index 96%
rename from tests/providers/amazon/aws/hooks/test_aws_dynamodb.py
rename to tests/providers/amazon/aws/hooks/test_dynamodb.py
index e191e3f..7377c28 100644
--- a/tests/providers/amazon/aws/hooks/test_aws_dynamodb.py
+++ b/tests/providers/amazon/aws/hooks/test_dynamodb.py
@@ -20,7 +20,7 @@
import unittest
import uuid
-from airflow.providers.amazon.aws.hooks.aws_dynamodb import AwsDynamoDBHook
+from airflow.providers.amazon.aws.hooks.dynamodb import AwsDynamoDBHook
try:
from moto import mock_dynamodb2
diff --git a/tests/providers/amazon/aws/transfers/test_hive_to_dynamodb.py b/tests/providers/amazon/aws/transfers/test_hive_to_dynamodb.py
index 9e1adcd..9a869c1 100644
--- a/tests/providers/amazon/aws/transfers/test_hive_to_dynamodb.py
+++ b/tests/providers/amazon/aws/transfers/test_hive_to_dynamodb.py
@@ -26,7 +26,7 @@ import pandas as pd
import airflow.providers.amazon.aws.transfers.hive_to_dynamodb
from airflow.models.dag import DAG
-from airflow.providers.amazon.aws.hooks.aws_dynamodb import AwsDynamoDBHook
+from airflow.providers.amazon.aws.hooks.dynamodb import AwsDynamoDBHook
DEFAULT_DATE = datetime.datetime(2015, 1, 1)
DEFAULT_DATE_ISO = DEFAULT_DATE.isoformat()