You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by GitBox <gi...@apache.org> on 2020/08/14 19:32:38 UTC
[GitHub] [airflow] feluelle commented on a change in pull request #8701: Adding ElastiCache Hook for creating, describing and deleting replication groups
feluelle commented on a change in pull request #8701:
URL: https://github.com/apache/airflow/pull/8701#discussion_r470795620
##########
File path: tests/providers/amazon/aws/hooks/test_elasticache_replication_group.py
##########
@@ -0,0 +1,357 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from unittest import TestCase
+from unittest.mock import Mock
+
+from airflow.exceptions import AirflowException
+from airflow.providers.amazon.aws.hooks.elasticache_replication_group import ElastiCacheReplicationGroupHook
+
+
+class TestElastiCacheHook(TestCase):
+ """
+ Test ElastiCacheHook
+ """
+ REPLICATION_GROUP_ID = "test-elasticache-hook"
+
+ REPLICATION_GROUP_CONFIG = {
+ 'ReplicationGroupId': REPLICATION_GROUP_ID,
+ 'ReplicationGroupDescription': REPLICATION_GROUP_ID,
+ 'AutomaticFailoverEnabled': False,
+ 'NumCacheClusters': 1,
+ 'CacheNodeType': 'cache.m5.large',
+ 'Engine': 'redis',
+ 'EngineVersion': '5.0.4',
+ 'CacheParameterGroupName': 'default.redis5.0'
+ }
+
+ VALID_STATES = frozenset({
+ 'creating', 'available', 'modifying', 'deleting', 'create - failed', 'snapshotting'
+ })
+
+ # Track calls to describe when deleting replication group
+ # First call will return status as `available` and we will initiate delete
+ # Second call with return status as `deleting`
+ # Subsequent call will raise ReplicationGroupNotFoundFault exception
+ describe_call_count_for_delete = 0
+
+ def setUp(self):
+ self.hook = ElastiCacheReplicationGroupHook()
+ setattr(self.hook, 'conn', Mock())
+
+ # We need this for every test
+ self.hook.conn.create_replication_group.return_value = {
+ "ReplicationGroup": {
+ "ReplicationGroupId": self.REPLICATION_GROUP_ID,
+ "Status": "creating"
+ }
+ }
+
+ def _create_replication_group(self):
+ return self.hook.create_replication_group(config=self.REPLICATION_GROUP_CONFIG)
+
+ def test_get_conn_not_none(self):
+ """
+ Test connection is not None
+ """
+ self.assertIsNotNone(self.hook.conn)
Review comment:
```suggestion
assert self.hook.conn is not None
```
##########
File path: airflow/providers/amazon/aws/hooks/elasticache_replication_group.py
##########
@@ -0,0 +1,289 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from time import sleep
+
+from airflow.exceptions import AirflowException
+from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
+
+
+class ElastiCacheReplicationGroupHook(AwsBaseHook):
+ """
+ Interact with AWS ElastiCache
+ """
+
+ TERMINAL_STATES = frozenset({"available", "create-failed", "deleting"})
+
+ def __init__(
+ self, max_retries=10, exponential_back_off_factor=1, initial_poke_interval=60, *args, **kwargs
+ ):
+ """
+ :param max_retries: Max retries for checking availability of and deleting replication group
+ :type max_retries: int
+ :param exponential_back_off_factor: Factor for deciding next sleep time
+ :type exponential_back_off_factor: float
+ :param initial_poke_interval: Initial sleep time in seconds
+ :type initial_poke_interval: float
+ """
+ self.max_retries = max_retries
+ self.exponential_back_off_factor = exponential_back_off_factor
+ self.initial_poke_interval = initial_poke_interval
+
+ super().__init__(client_type='elasticache', *args, **kwargs)
+
+ def create_replication_group(self, config):
+ """
+ Call ElastiCache API for creating a replication group
+
+ :param config: Configuration for creating the replication group
+ :type config: dict
+ :return: Response from ElastiCache create replication group API
+ :rtype: dict
+ """
+ return self.conn.create_replication_group(**config)
+
+ def delete_replication_group(self, replication_group_id):
+ """
+ Call ElastiCache API for deleting a replication group
+
+ :param replication_group_id: ID of replication group to delete
+ :type replication_group_id: str
+ :return: Response from ElastiCache delete replication group API
+ :rtype: dict
+ """
+ return self.conn.delete_replication_group(ReplicationGroupId=replication_group_id)
+
+ def describe_replication_group(self, replication_group_id):
+ """
+ Call ElastiCache API for describing a replication group
+
+ :param replication_group_id: ID of replication group to describe
+ :type replication_group_id: str
+ :return: Response from ElastiCache describe replication group API
+ :rtype: dict
+ """
+ return self.conn.describe_replication_groups(ReplicationGroupId=replication_group_id)
+
+ def get_replication_group_status(self, replication_group_id):
+ """
+ Get current status of replication group
+
+ :param replication_group_id: ID of replication group to check for status
+ :type replication_group_id: str
+ :return: Current status of replication group
+ :rtype: str
+ """
+ return self.describe_replication_group(replication_group_id)['ReplicationGroups'][0]['Status']
+
+ def is_replication_group_available(self, replication_group_id):
+ """
+ Helper for checking if replication group is available or not
+
+ :param replication_group_id: ID of replication group to check for availability
+ :type replication_group_id: str
+ :return: True if available else False
+ :rtype: bool
+ """
+ return self.get_replication_group_status(replication_group_id) == 'available'
+
+ def _has_reached_terminal_state(self, replication_group_id):
+ """
+ Helper for checking if we should stop poking replication group for availability or not
+
+ :param replication_group_id: ID of replication group to check for terminal state
+ :type replication_group_id: str
+ :return: Flag to check if availability-check should be stopped or not and current status
+ :rtype: (bool, str)
+ """
+ status = self.get_replication_group_status(replication_group_id=replication_group_id)
+
+ return status in self.TERMINAL_STATES, status
Review comment:
```suggestion
```
WDYT of removing this and being explicit about it (see line 146 & 147)
##########
File path: tests/providers/amazon/aws/hooks/test_elasticache_replication_group.py
##########
@@ -0,0 +1,357 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from unittest import TestCase
+from unittest.mock import Mock
+
+from airflow.exceptions import AirflowException
+from airflow.providers.amazon.aws.hooks.elasticache_replication_group import ElastiCacheReplicationGroupHook
+
+
+class TestElastiCacheHook(TestCase):
+ """
+ Test ElastiCacheHook
+ """
Review comment:
```suggestion
```
You don't need doc strings for test classes I think. However if you still want to document it, please add more meaningful message.
##########
File path: tests/providers/amazon/aws/hooks/test_elasticache_replication_group.py
##########
@@ -0,0 +1,357 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from unittest import TestCase
+from unittest.mock import Mock
+
+from airflow.exceptions import AirflowException
+from airflow.providers.amazon.aws.hooks.elasticache_replication_group import ElastiCacheReplicationGroupHook
+
+
+class TestElastiCacheHook(TestCase):
+ """
+ Test ElastiCacheHook
+ """
+ REPLICATION_GROUP_ID = "test-elasticache-hook"
+
+ REPLICATION_GROUP_CONFIG = {
+ 'ReplicationGroupId': REPLICATION_GROUP_ID,
+ 'ReplicationGroupDescription': REPLICATION_GROUP_ID,
+ 'AutomaticFailoverEnabled': False,
+ 'NumCacheClusters': 1,
+ 'CacheNodeType': 'cache.m5.large',
+ 'Engine': 'redis',
+ 'EngineVersion': '5.0.4',
+ 'CacheParameterGroupName': 'default.redis5.0'
+ }
+
+ VALID_STATES = frozenset({
+ 'creating', 'available', 'modifying', 'deleting', 'create - failed', 'snapshotting'
+ })
+
+ # Track calls to describe when deleting replication group
+ # First call will return status as `available` and we will initiate delete
+ # Second call with return status as `deleting`
+ # Subsequent call will raise ReplicationGroupNotFoundFault exception
+ describe_call_count_for_delete = 0
+
+ def setUp(self):
+ self.hook = ElastiCacheReplicationGroupHook()
+ setattr(self.hook, 'conn', Mock())
+
+ # We need this for every test
+ self.hook.conn.create_replication_group.return_value = {
+ "ReplicationGroup": {
+ "ReplicationGroupId": self.REPLICATION_GROUP_ID,
+ "Status": "creating"
+ }
+ }
+
+ def _create_replication_group(self):
+ return self.hook.create_replication_group(config=self.REPLICATION_GROUP_CONFIG)
+
+ def test_get_conn_not_none(self):
+ """
+ Test connection is not None
+ """
Review comment:
```suggestion
```
Same for the methods.
##########
File path: docs/operators-and-hooks-ref.rst
##########
@@ -446,6 +446,12 @@ These integrations allow you to perform various operations within the Amazon Web
- :mod:`airflow.providers.amazon.aws.operators.ecs`
-
+ * - `Amazon ElastiCache <https://aws.amazon.com/elasticache/redis//>`__
+ -
+ -
+ - :mod:`airflow.providers.amazon.aws.hooks.elasticache_replication_group`
Review comment:
```suggestion
- :mod:`airflow.providers.amazon.aws.hooks.elasticache_replication_group`
-
```
##########
File path: tests/providers/amazon/aws/hooks/test_elasticache_replication_group.py
##########
@@ -0,0 +1,357 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from unittest import TestCase
+from unittest.mock import Mock
+
+from airflow.exceptions import AirflowException
+from airflow.providers.amazon.aws.hooks.elasticache_replication_group import ElastiCacheReplicationGroupHook
+
+
+class TestElastiCacheHook(TestCase):
+ """
+ Test ElastiCacheHook
+ """
+ REPLICATION_GROUP_ID = "test-elasticache-hook"
+
+ REPLICATION_GROUP_CONFIG = {
+ 'ReplicationGroupId': REPLICATION_GROUP_ID,
+ 'ReplicationGroupDescription': REPLICATION_GROUP_ID,
+ 'AutomaticFailoverEnabled': False,
+ 'NumCacheClusters': 1,
+ 'CacheNodeType': 'cache.m5.large',
+ 'Engine': 'redis',
+ 'EngineVersion': '5.0.4',
+ 'CacheParameterGroupName': 'default.redis5.0'
+ }
+
+ VALID_STATES = frozenset({
+ 'creating', 'available', 'modifying', 'deleting', 'create - failed', 'snapshotting'
+ })
+
+ # Track calls to describe when deleting replication group
+ # First call will return status as `available` and we will initiate delete
+ # Second call with return status as `deleting`
+ # Subsequent call will raise ReplicationGroupNotFoundFault exception
+ describe_call_count_for_delete = 0
+
+ def setUp(self):
+ self.hook = ElastiCacheReplicationGroupHook()
+ setattr(self.hook, 'conn', Mock())
Review comment:
```suggestion
self.hook.conn = Mock()
```
##########
File path: airflow/providers/amazon/aws/hooks/elasticache_replication_group.py
##########
@@ -0,0 +1,289 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from time import sleep
+
+from airflow.exceptions import AirflowException
+from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
+
+
+class ElastiCacheReplicationGroupHook(AwsBaseHook):
+ """
+ Interact with AWS ElastiCache
+ """
+
+ TERMINAL_STATES = frozenset({"available", "create-failed", "deleting"})
+
+ def __init__(
+ self, max_retries=10, exponential_back_off_factor=1, initial_poke_interval=60, *args, **kwargs
+ ):
+ """
+ :param max_retries: Max retries for checking availability of and deleting replication group
+ :type max_retries: int
+ :param exponential_back_off_factor: Factor for deciding next sleep time
+ :type exponential_back_off_factor: float
+ :param initial_poke_interval: Initial sleep time in seconds
+ :type initial_poke_interval: float
+ """
+ self.max_retries = max_retries
+ self.exponential_back_off_factor = exponential_back_off_factor
+ self.initial_poke_interval = initial_poke_interval
+
+ super().__init__(client_type='elasticache', *args, **kwargs)
+
+ def create_replication_group(self, config):
+ """
+ Call ElastiCache API for creating a replication group
+
+ :param config: Configuration for creating the replication group
+ :type config: dict
+ :return: Response from ElastiCache create replication group API
+ :rtype: dict
+ """
+ return self.conn.create_replication_group(**config)
+
+ def delete_replication_group(self, replication_group_id):
+ """
+ Call ElastiCache API for deleting a replication group
+
+ :param replication_group_id: ID of replication group to delete
+ :type replication_group_id: str
+ :return: Response from ElastiCache delete replication group API
+ :rtype: dict
+ """
+ return self.conn.delete_replication_group(ReplicationGroupId=replication_group_id)
+
+ def describe_replication_group(self, replication_group_id):
+ """
+ Call ElastiCache API for describing a replication group
+
+ :param replication_group_id: ID of replication group to describe
+ :type replication_group_id: str
+ :return: Response from ElastiCache describe replication group API
+ :rtype: dict
+ """
+ return self.conn.describe_replication_groups(ReplicationGroupId=replication_group_id)
+
+ def get_replication_group_status(self, replication_group_id):
+ """
+ Get current status of replication group
+
+ :param replication_group_id: ID of replication group to check for status
+ :type replication_group_id: str
+ :return: Current status of replication group
+ :rtype: str
+ """
+ return self.describe_replication_group(replication_group_id)['ReplicationGroups'][0]['Status']
+
+ def is_replication_group_available(self, replication_group_id):
+ """
+ Helper for checking if replication group is available or not
+
+ :param replication_group_id: ID of replication group to check for availability
+ :type replication_group_id: str
+ :return: True if available else False
+ :rtype: bool
+ """
+ return self.get_replication_group_status(replication_group_id) == 'available'
+
+ def _has_reached_terminal_state(self, replication_group_id):
+ """
+ Helper for checking if we should stop poking replication group for availability or not
+
+ :param replication_group_id: ID of replication group to check for terminal state
+ :type replication_group_id: str
+ :return: Flag to check if availability-check should be stopped or not and current status
+ :rtype: (bool, str)
+ """
+ status = self.get_replication_group_status(replication_group_id=replication_group_id)
+
+ return status in self.TERMINAL_STATES, status
+
+ def wait_for_availability(
+ self,
+ replication_group_id,
+ initial_sleep_time=None,
+ exponential_back_off_factor=None,
+ max_retries=None
+ ):
+ """
+ Check if replication group is available or not by performing a describe over it
+
+ :param max_retries: Max retries for checking availability of replication group
+ :type max_retries: int
+ :param exponential_back_off_factor: Factor for deciding next sleep time
+ :type exponential_back_off_factor: float
+ :param initial_sleep_time: Initial sleep time in seconds
+ :type initial_sleep_time: float
+ :param replication_group_id: ID of replication group to check for availability
+ :type replication_group_id: str
+ :return: True if replication is available else False
+ :rtype: bool
+ """
+ sleep_time = initial_sleep_time or self.initial_poke_interval
+ exponential_back_off_factor = exponential_back_off_factor or self.exponential_back_off_factor
+ max_retries = max_retries or self.max_retries
+ num_tries = 0
+ status = 'not-found'
+ stop_poking = False
+
+ while not stop_poking and num_tries <= max_retries:
+ stop_poking, status = self._has_reached_terminal_state(replication_group_id=replication_group_id)
+
Review comment:
```suggestion
status = self.get_replication_group_status(replication_group_id=replication_group_id)
stop_poking = status in self.TERMINAL_STATES
```
##########
File path: airflow/providers/amazon/aws/hooks/elasticache_replication_group.py
##########
@@ -0,0 +1,289 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from time import sleep
+
+from airflow.exceptions import AirflowException
+from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
+
+
+class ElastiCacheReplicationGroupHook(AwsBaseHook):
+ """
+ Interact with AWS ElastiCache
+ """
+
+ TERMINAL_STATES = frozenset({"available", "create-failed", "deleting"})
+
+ def __init__(
+ self, max_retries=10, exponential_back_off_factor=1, initial_poke_interval=60, *args, **kwargs
+ ):
+ """
+ :param max_retries: Max retries for checking availability of and deleting replication group
+ :type max_retries: int
+ :param exponential_back_off_factor: Factor for deciding next sleep time
+ :type exponential_back_off_factor: float
+ :param initial_poke_interval: Initial sleep time in seconds
+ :type initial_poke_interval: float
+ """
+ self.max_retries = max_retries
+ self.exponential_back_off_factor = exponential_back_off_factor
+ self.initial_poke_interval = initial_poke_interval
+
+ super().__init__(client_type='elasticache', *args, **kwargs)
+
+ def create_replication_group(self, config):
+ """
+ Call ElastiCache API for creating a replication group
+
+ :param config: Configuration for creating the replication group
+ :type config: dict
+ :return: Response from ElastiCache create replication group API
+ :rtype: dict
+ """
+ return self.conn.create_replication_group(**config)
+
+ def delete_replication_group(self, replication_group_id):
+ """
+ Call ElastiCache API for deleting a replication group
+
+ :param replication_group_id: ID of replication group to delete
+ :type replication_group_id: str
+ :return: Response from ElastiCache delete replication group API
+ :rtype: dict
+ """
+ return self.conn.delete_replication_group(ReplicationGroupId=replication_group_id)
+
+ def describe_replication_group(self, replication_group_id):
+ """
+ Call ElastiCache API for describing a replication group
+
+ :param replication_group_id: ID of replication group to describe
+ :type replication_group_id: str
+ :return: Response from ElastiCache describe replication group API
+ :rtype: dict
+ """
+ return self.conn.describe_replication_groups(ReplicationGroupId=replication_group_id)
+
+ def get_replication_group_status(self, replication_group_id):
+ """
+ Get current status of replication group
+
+ :param replication_group_id: ID of replication group to check for status
+ :type replication_group_id: str
+ :return: Current status of replication group
+ :rtype: str
+ """
+ return self.describe_replication_group(replication_group_id)['ReplicationGroups'][0]['Status']
+
+ def is_replication_group_available(self, replication_group_id):
+ """
+ Helper for checking if replication group is available or not
+
+ :param replication_group_id: ID of replication group to check for availability
+ :type replication_group_id: str
+ :return: True if available else False
+ :rtype: bool
+ """
+ return self.get_replication_group_status(replication_group_id) == 'available'
+
+ def _has_reached_terminal_state(self, replication_group_id):
+ """
+ Helper for checking if we should stop poking replication group for availability or not
+
+ :param replication_group_id: ID of replication group to check for terminal state
+ :type replication_group_id: str
+ :return: Flag to check if availability-check should be stopped or not and current status
+ :rtype: (bool, str)
+ """
+ status = self.get_replication_group_status(replication_group_id=replication_group_id)
+
+ return status in self.TERMINAL_STATES, status
+
+ def wait_for_availability(
+ self,
+ replication_group_id,
+ initial_sleep_time=None,
+ exponential_back_off_factor=None,
+ max_retries=None
+ ):
+ """
+ Check if replication group is available or not by performing a describe over it
+
+ :param max_retries: Max retries for checking availability of replication group
+ :type max_retries: int
+ :param exponential_back_off_factor: Factor for deciding next sleep time
+ :type exponential_back_off_factor: float
+ :param initial_sleep_time: Initial sleep time in seconds
+ :type initial_sleep_time: float
+ :param replication_group_id: ID of replication group to check for availability
+ :type replication_group_id: str
+ :return: True if replication is available else False
+ :rtype: bool
+ """
+ sleep_time = initial_sleep_time or self.initial_poke_interval
+ exponential_back_off_factor = exponential_back_off_factor or self.exponential_back_off_factor
+ max_retries = max_retries or self.max_retries
+ num_tries = 0
+ status = 'not-found'
+ stop_poking = False
+
+ while not stop_poking and num_tries <= max_retries:
+ stop_poking, status = self._has_reached_terminal_state(replication_group_id=replication_group_id)
+
+ self.log.info(
+ 'Current status of replication group with ID %s is %s',
+ replication_group_id,
+ status
+ )
+
+ if not stop_poking:
+ num_tries += 1
+
+ # No point in sleeping if all tries have exhausted
+ if num_tries > max_retries:
+ break
+
+ self.log.info('Poke retry %s. Sleep time %s seconds. Sleeping...', num_tries, sleep_time)
+
+ sleep(sleep_time)
+
+ sleep_time = sleep_time * exponential_back_off_factor
Review comment:
```suggestion
sleep_time *= exponential_back_off_factor
```
##########
File path: airflow/providers/amazon/aws/hooks/elasticache_replication_group.py
##########
@@ -0,0 +1,289 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from time import sleep
+
+from airflow.exceptions import AirflowException
+from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
+
+
+class ElastiCacheReplicationGroupHook(AwsBaseHook):
+ """
+ Interact with AWS ElastiCache
+ """
+
+ TERMINAL_STATES = frozenset({"available", "create-failed", "deleting"})
+
+ def __init__(
+ self, max_retries=10, exponential_back_off_factor=1, initial_poke_interval=60, *args, **kwargs
+ ):
+ """
+ :param max_retries: Max retries for checking availability of and deleting replication group
+ :type max_retries: int
+ :param exponential_back_off_factor: Factor for deciding next sleep time
+ :type exponential_back_off_factor: float
+ :param initial_poke_interval: Initial sleep time in seconds
+ :type initial_poke_interval: float
+ """
+ self.max_retries = max_retries
+ self.exponential_back_off_factor = exponential_back_off_factor
+ self.initial_poke_interval = initial_poke_interval
+
+ super().__init__(client_type='elasticache', *args, **kwargs)
+
+ def create_replication_group(self, config):
+ """
+ Call ElastiCache API for creating a replication group
+
+ :param config: Configuration for creating the replication group
+ :type config: dict
+ :return: Response from ElastiCache create replication group API
+ :rtype: dict
+ """
+ return self.conn.create_replication_group(**config)
+
+ def delete_replication_group(self, replication_group_id):
+ """
+ Call ElastiCache API for deleting a replication group
+
+ :param replication_group_id: ID of replication group to delete
+ :type replication_group_id: str
+ :return: Response from ElastiCache delete replication group API
+ :rtype: dict
+ """
+ return self.conn.delete_replication_group(ReplicationGroupId=replication_group_id)
+
+ def describe_replication_group(self, replication_group_id):
+ """
+ Call ElastiCache API for describing a replication group
+
+ :param replication_group_id: ID of replication group to describe
+ :type replication_group_id: str
+ :return: Response from ElastiCache describe replication group API
+ :rtype: dict
+ """
+ return self.conn.describe_replication_groups(ReplicationGroupId=replication_group_id)
+
+ def get_replication_group_status(self, replication_group_id):
+ """
+ Get current status of replication group
+
+ :param replication_group_id: ID of replication group to check for status
+ :type replication_group_id: str
+ :return: Current status of replication group
+ :rtype: str
+ """
+ return self.describe_replication_group(replication_group_id)['ReplicationGroups'][0]['Status']
+
+ def is_replication_group_available(self, replication_group_id):
+ """
+ Helper for checking if replication group is available or not
+
+ :param replication_group_id: ID of replication group to check for availability
+ :type replication_group_id: str
+ :return: True if available else False
+ :rtype: bool
+ """
+ return self.get_replication_group_status(replication_group_id) == 'available'
+
+ def _has_reached_terminal_state(self, replication_group_id):
+ """
+ Helper for checking if we should stop poking replication group for availability or not
+
+ :param replication_group_id: ID of replication group to check for terminal state
+ :type replication_group_id: str
+ :return: Flag to check if availability-check should be stopped or not and current status
+ :rtype: (bool, str)
+ """
+ status = self.get_replication_group_status(replication_group_id=replication_group_id)
+
+ return status in self.TERMINAL_STATES, status
Review comment:
Or pass status to this function and return only if it is a terminal state. Returning two values where one depends on the other one looks to me that this should be two statements like I suggested in line 146 & 147.
##########
File path: airflow/providers/amazon/aws/hooks/elasticache_replication_group.py
##########
@@ -0,0 +1,289 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from time import sleep
+
+from airflow.exceptions import AirflowException
+from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
+
+
+class ElastiCacheReplicationGroupHook(AwsBaseHook):
+ """
+ Interact with AWS ElastiCache
+ """
+
+ TERMINAL_STATES = frozenset({"available", "create-failed", "deleting"})
+
+ def __init__(
+ self, max_retries=10, exponential_back_off_factor=1, initial_poke_interval=60, *args, **kwargs
+ ):
+ """
+ :param max_retries: Max retries for checking availability of and deleting replication group
+ :type max_retries: int
+ :param exponential_back_off_factor: Factor for deciding next sleep time
+ :type exponential_back_off_factor: float
+ :param initial_poke_interval: Initial sleep time in seconds
+ :type initial_poke_interval: float
+ """
+ self.max_retries = max_retries
+ self.exponential_back_off_factor = exponential_back_off_factor
+ self.initial_poke_interval = initial_poke_interval
+
+ super().__init__(client_type='elasticache', *args, **kwargs)
+
+ def create_replication_group(self, config):
+ """
+ Call ElastiCache API for creating a replication group
+
+ :param config: Configuration for creating the replication group
+ :type config: dict
+ :return: Response from ElastiCache create replication group API
+ :rtype: dict
+ """
+ return self.conn.create_replication_group(**config)
+
+ def delete_replication_group(self, replication_group_id):
+ """
+ Call ElastiCache API for deleting a replication group
+
+ :param replication_group_id: ID of replication group to delete
+ :type replication_group_id: str
+ :return: Response from ElastiCache delete replication group API
+ :rtype: dict
+ """
+ return self.conn.delete_replication_group(ReplicationGroupId=replication_group_id)
+
+ def describe_replication_group(self, replication_group_id):
+ """
+ Call ElastiCache API for describing a replication group
+
+ :param replication_group_id: ID of replication group to describe
+ :type replication_group_id: str
+ :return: Response from ElastiCache describe replication group API
+ :rtype: dict
+ """
+ return self.conn.describe_replication_groups(ReplicationGroupId=replication_group_id)
+
+ def get_replication_group_status(self, replication_group_id):
+ """
+ Get current status of replication group
+
+ :param replication_group_id: ID of replication group to check for status
+ :type replication_group_id: str
+ :return: Current status of replication group
+ :rtype: str
+ """
+ return self.describe_replication_group(replication_group_id)['ReplicationGroups'][0]['Status']
+
+ def is_replication_group_available(self, replication_group_id):
+ """
+ Helper for checking if replication group is available or not
+
+ :param replication_group_id: ID of replication group to check for availability
+ :type replication_group_id: str
+ :return: True if available else False
+ :rtype: bool
+ """
+ return self.get_replication_group_status(replication_group_id) == 'available'
+
+ def _has_reached_terminal_state(self, replication_group_id):
+ """
+ Helper for checking if we should stop poking replication group for availability or not
+
+ :param replication_group_id: ID of replication group to check for terminal state
+ :type replication_group_id: str
+ :return: Flag to check if availability-check should be stopped or not and current status
+ :rtype: (bool, str)
+ """
+ status = self.get_replication_group_status(replication_group_id=replication_group_id)
+
+ return status in self.TERMINAL_STATES, status
+
+ def wait_for_availability(
+ self,
+ replication_group_id,
+ initial_sleep_time=None,
+ exponential_back_off_factor=None,
+ max_retries=None
+ ):
+ """
+ Check if replication group is available or not by performing a describe over it
+
+ :param max_retries: Max retries for checking availability of replication group
+ :type max_retries: int
+ :param exponential_back_off_factor: Factor for deciding next sleep time
+ :type exponential_back_off_factor: float
+ :param initial_sleep_time: Initial sleep time in seconds
+ :type initial_sleep_time: float
+ :param replication_group_id: ID of replication group to check for availability
+ :type replication_group_id: str
+ :return: True if replication is available else False
+ :rtype: bool
+ """
+ sleep_time = initial_sleep_time or self.initial_poke_interval
+ exponential_back_off_factor = exponential_back_off_factor or self.exponential_back_off_factor
+ max_retries = max_retries or self.max_retries
+ num_tries = 0
+ status = 'not-found'
+ stop_poking = False
+
+ while not stop_poking and num_tries <= max_retries:
+ stop_poking, status = self._has_reached_terminal_state(replication_group_id=replication_group_id)
+
+ self.log.info(
+ 'Current status of replication group with ID %s is %s',
+ replication_group_id,
+ status
+ )
+
+ if not stop_poking:
+ num_tries += 1
+
+ # No point in sleeping if all tries have exhausted
+ if num_tries > max_retries:
+ break
+
+ self.log.info('Poke retry %s. Sleep time %s seconds. Sleeping...', num_tries, sleep_time)
+
+ sleep(sleep_time)
+
+ sleep_time = sleep_time * exponential_back_off_factor
+
+ if status != 'available':
+ self.log.warning('Replication group is not available. Current status is "%s"', status)
+
+ return False
+
+ return True
+
+ def wait_for_deletion(
+ self,
+ replication_group_id,
+ initial_sleep_time=None,
+ exponential_back_off_factor=None,
+ max_retries=None
+ ):
+ """
+ Helper for deleting a replication group ensuring it is either deleted or can't be deleted
+
+ :param replication_group_id: ID of replication to delete
+ :type replication_group_id: str
+ :param max_retries: Max retries for checking availability of replication group
+ :type max_retries: int
+ :param exponential_back_off_factor: Factor for deciding next sleep time
+ :type exponential_back_off_factor: float
+ :param initial_sleep_time: Initial sleep time in second
+ :type initial_sleep_time: float
+ :return: Response from ElastiCache delete replication group API and flag to identify if deleted or not
+ :rtype: (dict, bool)
+ """
+ deleted = False
+ sleep_time = initial_sleep_time or self.initial_poke_interval
+ exponential_back_off_factor = exponential_back_off_factor or self.exponential_back_off_factor
+ max_retries = max_retries or self.max_retries
+ num_tries = 0
+ response = None
+
+ while not deleted and num_tries <= max_retries:
+ try:
+ status = self.get_replication_group_status(replication_group_id=replication_group_id)
+
+ self.log.info(
+ 'Current status of replication group with ID %s is %s',
+ replication_group_id,
+ status
+ )
+
+ # Can only delete if status is `available`
+ # Status becomes `deleting` after this call so this will only run once
+ if status == 'available':
+ self.log.info("Initiating delete and then wait for it to finish")
+
+ response = self.delete_replication_group(replication_group_id=replication_group_id)
+
+ except self.conn.exceptions.ReplicationGroupNotFoundFault:
+ self.log.info("Replication group with ID '%s' does not exist", replication_group_id)
+
+ deleted = True
+
+ # This should never occur as we only issue a delete request when status is `available`
+ # which is a valid status for deletion. Still handling for safety.
+ except self.conn.exceptions.InvalidReplicationGroupStateFault as exp:
+ # status Error Response
+ # creating - Cache cluster <cluster_id> is not in a valid state to be deleted.
+ # deleting - Replication group <replication_group_id> has status deleting which is not valid
+ # for deletion.
+ # modifying - Replication group <replication_group_id> has status deleting which is not valid
+ # for deletion.
+
+ message = exp.response['Error']['Message']
+
+ self.log.warning('Received error message from AWS ElastiCache API : %s', message)
+
+ if not deleted:
+ num_tries += 1
+
+ # No point in sleeping if all tries have exhausted
+ if num_tries > max_retries:
+ break
+
+ self.log.info('Poke retry %s. Sleep time %s seconds. Sleeping...', num_tries, sleep_time)
+
+ sleep(sleep_time)
+
+ sleep_time = sleep_time * exponential_back_off_factor
Review comment:
```suggestion
sleep_time *= exponential_back_off_factor
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org