You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@libcloud.apache.org by cl...@apache.org on 2021/02/10 22:17:09 UTC
[libcloud] branch trunk updated: Add support for Azure Government
to Azure Blobs storage driver (#1552)
This is an automated email from the ASF dual-hosted git repository.
clewolff pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/libcloud.git
The following commit(s) were added to refs/heads/trunk by this push:
new 86f5c1c Add support for Azure Government to Azure Blobs storage driver (#1552)
86f5c1c is described below
commit 86f5c1cc5f74a25cb50177a7a06b9ff62ef96ee7
Author: Clemens Wolff <cl...@apache.org>
AuthorDate: Wed Feb 10 17:16:59 2021 -0500
Add support for Azure Government to Azure Blobs storage driver (#1552)
* Fix typo in documentation
* Add support for blob storage in Azure Government
Note that this change also enables using the driver with Azure China and
Azure Private Link.
* Add unit tests for GovCloud host formatting
* Add unit tests for Azurite host formatting
* Allow conn_kwargs to override host
---
CHANGES.rst | 13 +++++++
docs/examples/storage/azure/instantiate_gov.py | 8 +++++
docs/storage/drivers/azure_blobs.rst | 16 +++++++++
libcloud/common/base.py | 20 ++++++-----
libcloud/storage/drivers/azure_blobs.py | 47 +++++++++++++++++++-------
libcloud/test/storage/test_azure_blobs.py | 29 ++++++++++++++++
6 files changed, 112 insertions(+), 21 deletions(-)
diff --git a/CHANGES.rst b/CHANGES.rst
index a99a8bd..f4d6301 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,6 +1,19 @@
Changelog
=========
+Changes in Apache Libcloud 3.3.2
+--------------------------------
+
+Storage
+~~~~~~~
+
+- [Azure Blobs] Enable the Azure storage driver to be used with
+ Azure Government, Azure China, and Azure Private Link by setting
+ the driver host argument to the endpoint suffix for the environment.
+
+ Reported by Melissa Kersh - @mkcello96
+ (GITHUB-1551)
+
Changes in Apache Libcloud 3.3.1
--------------------------------
diff --git a/docs/examples/storage/azure/instantiate_gov.py b/docs/examples/storage/azure/instantiate_gov.py
new file mode 100644
index 0000000..46a6c01
--- /dev/null
+++ b/docs/examples/storage/azure/instantiate_gov.py
@@ -0,0 +1,8 @@
+from libcloud.storage.types import Provider
+from libcloud.storage.providers import get_driver
+
+cls = get_driver(Provider.AZURE_BLOBS)
+
+driver = cls(key='your storage account name',
+ secret='your access key',
+ host='blob.core.usgovcloudapi.net')
diff --git a/docs/storage/drivers/azure_blobs.rst b/docs/storage/drivers/azure_blobs.rst
index 824889c..956b3f1 100644
--- a/docs/storage/drivers/azure_blobs.rst
+++ b/docs/storage/drivers/azure_blobs.rst
@@ -33,6 +33,19 @@ below.
.. literalinclude:: /examples/storage/azure/instantiate.py
:language: python
+Connecting to Azure Government
+------------------------------
+
+To target an `Azure Government`_ storage account, you can instantiate the driver
+by setting a custom storage host argument as shown below.
+
+.. literalinclude:: /examples/storage/azure/instantiate_gov.py
+ :language: python
+
+Setting a custom host argument can also be leveraged to customize the blob
+endpoint and connect to a storage account in `Azure China`_ or
+`Azure Private Link`_.
+
Connecting to self-hosted Azure Storage implementations
-------------------------------------------------------
@@ -51,3 +64,6 @@ Azure Storage implementations such as `Azure Blob Storage on IoT Edge`_.
.. _`BlockBlobStorage accounts`: https://docs.microsoft.com/en-us/azure/storage/common/storage-account-overview#blockblobstorage-accounts
.. _`Azurite storage emulator`: https://github.com/Azure/Azurite
.. _`Azure Blob Storage on IoT Edge`: https://docs.microsoft.com/en-us/azure/iot-edge/how-to-store-data-blob
+.. _`Azure Government`: https://docs.microsoft.com/en-us/azure/azure-government/documentation-government-developer-guide
+.. _`Azure China`: https://docs.microsoft.com/en-us/azure/china/resources-developer-guide
+.. _`Azure Private Link`: https://docs.microsoft.com/en-us/azure/private-link/private-link-overview
diff --git a/libcloud/common/base.py b/libcloud/common/base.py
index 48b0e22..b944107 100644
--- a/libcloud/common/base.py
+++ b/libcloud/common/base.py
@@ -984,6 +984,15 @@ class BaseDriver(object):
self.key = key
self.secret = secret
self.secure = secure
+ self.api_version = api_version
+ self.region = region
+
+ conn_kwargs = self._ex_connection_class_kwargs()
+ conn_kwargs.update({'timeout': kwargs.pop('timeout', None),
+ 'retry_delay': kwargs.pop('retry_delay', None),
+ 'backoff': kwargs.pop('backoff', None),
+ 'proxy_url': kwargs.pop('proxy_url', None)})
+
args = [self.key]
if self.secret is not None:
@@ -991,22 +1000,15 @@ class BaseDriver(object):
args.append(secure)
+ host = conn_kwargs.pop('host', None) or host
+
if host is not None:
args.append(host)
if port is not None:
args.append(port)
- self.api_version = api_version
- self.region = region
-
- conn_kwargs = self._ex_connection_class_kwargs()
- conn_kwargs.update({'timeout': kwargs.pop('timeout', None),
- 'retry_delay': kwargs.pop('retry_delay', None),
- 'backoff': kwargs.pop('backoff', None),
- 'proxy_url': kwargs.pop('proxy_url', None)})
self.connection = self.connectionCls(*args, **conn_kwargs)
-
self.connection.driver = self
self.connection.connect()
diff --git a/libcloud/storage/drivers/azure_blobs.py b/libcloud/storage/drivers/azure_blobs.py
index 3393784..6c65c51 100644
--- a/libcloud/storage/drivers/azure_blobs.py
+++ b/libcloud/storage/drivers/azure_blobs.py
@@ -68,6 +68,9 @@ AZURE_LEASE_PERIOD = int(
)
AZURE_STORAGE_HOST_SUFFIX = 'blob.core.windows.net'
+AZURE_STORAGE_HOST_SUFFIX_CHINA = 'blob.core.chinacloudapi.cn'
+AZURE_STORAGE_HOST_SUFFIX_GOVERNMENT = 'blob.core.usgovcloudapi.net'
+AZURE_STORAGE_HOST_SUFFIX_PRIVATELINK = 'privatelink.blob.core.windows.net'
AZURE_STORAGE_CDN_URL_DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
@@ -173,11 +176,12 @@ class AzureBlobsConnection(AzureConnection):
these deployments, the parameter ``account_prefix`` must be set on the
connection. This is done by instantiating the driver with arguments such
as ``host='somewhere.tld'`` and ``key='theaccount'``. To specify a custom
- host without an account prefix, e.g. for use-cases where the custom host
- implements an auditing proxy or similar, the driver can be instantiated
- with ``host='theaccount.somewhere.tld'`` and ``key=''``.
+ host without an account prefix, e.g. to connect to Azure Government or
+ Azure China, the driver can be instantiated with the appropriate storage
+ endpoint suffix, e.g. ``host='blob.core.usgovcloudapi.net'`` and
+ ``key='theaccount'``.
- :param account_prefix: Optional prefix identifying the sotrage account.
+ :param account_prefix: Optional prefix identifying the storage account.
Used when connecting to a custom deployment of the
storage service like Azurite or IoT Edge Storage.
:type account_prefix: ``str``
@@ -206,7 +210,7 @@ class AzureBlobsStorageDriver(StorageDriver):
def __init__(self, key, secret=None, secure=True, host=None, port=None,
**kwargs):
- self._host_argument_set = bool(host)
+ self._host = host
# B64decode() this key and keep it, so that we don't have to do
# so for every request. Minor performance improvement
@@ -217,15 +221,34 @@ class AzureBlobsStorageDriver(StorageDriver):
port=port, **kwargs)
def _ex_connection_class_kwargs(self):
- result = {}
-
- # host argument has precedence
- if not self._host_argument_set:
- result['host'] = '%s.%s' % (self.key, AZURE_STORAGE_HOST_SUFFIX)
+ # if the user didn't provide a custom host value, assume we're
+ # targeting the default Azure Storage endpoints
+ if self._host is None:
+ return {'host': '%s.%s' % (self.key, AZURE_STORAGE_HOST_SUFFIX)}
+
+ # connecting to a special storage region like Azure Government or
+ # Azure China requires setting a custom storage endpoint but we
+ # still use the same scheme to identify a specific account as for
+ # the standard storage endpoint
+ try:
+ host_suffix = next(
+ host_suffix
+ for host_suffix in (
+ AZURE_STORAGE_HOST_SUFFIX_CHINA,
+ AZURE_STORAGE_HOST_SUFFIX_GOVERNMENT,
+ AZURE_STORAGE_HOST_SUFFIX_PRIVATELINK,
+ )
+ if self._host.endswith(host_suffix)
+ )
+ except StopIteration:
+ pass
else:
- result['account_prefix'] = self.key
+ return {'host': '%s.%s' % (self.key, host_suffix)}
- return result
+ # if the host isn't targeting one of the special storage regions, it
+ # must be pointing to Azurite or IoT Edge Storage so switch to prefix
+ # identification
+ return {'account_prefix': self.key}
def _xml_to_container(self, node):
"""
diff --git a/libcloud/test/storage/test_azure_blobs.py b/libcloud/test/storage/test_azure_blobs.py
index 9d3eb3c..19dfcee 100644
--- a/libcloud/test/storage/test_azure_blobs.py
+++ b/libcloud/test/storage/test_azure_blobs.py
@@ -935,6 +935,35 @@ class AzureBlobsTests(unittest.TestCase):
self.assertEqual(host2, 'fakeaccount2.blob.core.windows.net')
self.assertEqual(host3, 'test.foo.bar.com')
+ def test_storage_driver_host_govcloud(self):
+ driver1 = self.driver_type(
+ 'fakeaccount1', 'deadbeafcafebabe==',
+ host='blob.core.usgovcloudapi.net')
+ driver2 = self.driver_type(
+ 'fakeaccount2', 'deadbeafcafebabe==',
+ host='fakeaccount2.blob.core.usgovcloudapi.net')
+
+ host1 = driver1.connection.host
+ host2 = driver2.connection.host
+ account_prefix_1 = driver1.connection.account_prefix
+ account_prefix_2 = driver2.connection.account_prefix
+
+ self.assertEqual(host1, 'fakeaccount1.blob.core.usgovcloudapi.net')
+ self.assertEqual(host2, 'fakeaccount2.blob.core.usgovcloudapi.net')
+ self.assertIsNone(account_prefix_1)
+ self.assertIsNone(account_prefix_2)
+
+ def test_storage_driver_host_azurite(self):
+ driver = self.driver_type(
+ 'fakeaccount1', 'deadbeafcafebabe==',
+ host='localhost', port=10000, secure=False)
+
+ host = driver.connection.host
+ account_prefix = driver.connection.account_prefix
+
+ self.assertEqual(host, 'localhost')
+ self.assertEqual(account_prefix, 'fakeaccount1')
+
class AzuriteBlobsTests(AzureBlobsTests):
driver_args = STORAGE_AZURITE_BLOBS_PARAMS