You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@libcloud.apache.org by "Jack (JIRA)" <ji...@apache.org> on 2018/01/27 16:19:02 UTC
[jira] [Updated] (LIBCLOUD-979) libcloud.storage / S3_RGW: cannot fetch some objects

     [ https://issues.apache.org/jira/browse/LIBCLOUD-979?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]

Jack updated LIBCLOUD-979:
--------------------------
    Description: 
Libcloud cannot handle some legitimates objects
 Here is an example, that will explain the issue

Please check this code:
{quote}from libcloud.storage.types import Provider
 from libcloud.storage.providers import get_driver

driver = get_driver(Provider.S3_RGW)(key='access_key', secret='secret_key', host='hostname')

print(driver.get_object('tilde', 'file~'))
{quote}
 

Output:
{quote}~$ ./test.py 
 Traceback (most recent call last):
 File "./test.py", line 8, in <module>
 print(driver.get_object('tilde', 'file~'))
 File "/tmp/libcloud/libcloud/storage/drivers/s3.py", line 332, in get_object
 response = self.connection.request(object_path, method='HEAD')
 File "/tmp/libcloud/libcloud/common/base.py", line 637, in request
 response = responseCls(**kwargs)
 File "/tmp/libcloud/libcloud/common/base.py", line 152, in __init__
 message=self.parse_error(),
 File "/tmp/libcloud/libcloud/storage/drivers/s3.py", line 96, in parse_error
 raise InvalidCredsError(self.body)
 libcloud.common.types.InvalidCredsError: ''
{quote}
 

A tcpdump shows that libcloud tranforms the key from "file~" to "file%7E" (thus, url-encode)

The slight patch below fixes this issue:
{quote}diff --git a/libcloud/storage/drivers/s3.py b/libcloud/storage/drivers/s3.py
 index f9f4b6c9..7107a8c6 100644
 — a/libcloud/storage/drivers/s3.py
 +++ b/libcloud/storage/drivers/s3.py
 @@ -366,7 +366,7 @@ class BaseS3StorageDriver(StorageDriver):
 :rtype: ``str``
 """
 container_url = self._get_container_path(container)
 - object_name_cleaned = self._clean_object_name(object_name)
 + object_name_cleaned = object_name
 object_path = '%s/%s' % (container_url, object_name_cleaned)
 return object_path

@@ -776,9 +776,6 @@ class BaseS3StorageDriver(StorageDriver):
 delimiter=None):
 self._abort_multipart(container, upload.key, upload.id)
 - def _clean_object_name(self, name):
 - name = urlquote(name)
 - return name

def _put_object(self, container, object_name, method='PUT',
 query_args=None, extra=None, file_path=None,

~$ ./test.py 
 <Object: name=file~, size=0, hash=d41d8cd98f00b204e9800998ecf8427e, provider=Ceph RGW S3 (default) ...>
{quote}
 

This kind of issues existed on other projects (see [https://github.com/aws/aws-sdk-php/issues/37] and [https://github.com/aws/aws-sdk-go/issues/45] for instance)

I do not know if this is specific to Ceph, or if this issue applies to amazon's S3 as well (regarding the issues above, this seems to be)

 

Thanks

  was:
Libcloud cannot handle some legitimates object
 Here is an example, that will explain the issue

Please check this code:
{quote}from libcloud.storage.types import Provider
 from libcloud.storage.providers import get_driver

driver = get_driver(Provider.S3_RGW)(key='access_key', secret='secret_key', host='hostname')

print(driver.get_object('tilde', 'file~'))
{quote}
 

Output:
{quote}~$ ./test.py 
Traceback (most recent call last):
 File "./test.py", line 8, in <module>
 print(driver.get_object('tilde', 'file~'))
 File "/tmp/libcloud/libcloud/storage/drivers/s3.py", line 332, in get_object
 response = self.connection.request(object_path, method='HEAD')
 File "/tmp/libcloud/libcloud/common/base.py", line 637, in request
 response = responseCls(**kwargs)
 File "/tmp/libcloud/libcloud/common/base.py", line 152, in __init__
 message=self.parse_error(),
 File "/tmp/libcloud/libcloud/storage/drivers/s3.py", line 96, in parse_error
 raise InvalidCredsError(self.body)
libcloud.common.types.InvalidCredsError: ''
{quote}
 

A tcpdump shows that libcloud tranforms the key from "file~" to "file%7E" (thus, url-encode)

The slight patch below fixes this issue:
{quote}diff --git a/libcloud/storage/drivers/s3.py b/libcloud/storage/drivers/s3.py
index f9f4b6c9..7107a8c6 100644
--- a/libcloud/storage/drivers/s3.py
+++ b/libcloud/storage/drivers/s3.py
@@ -366,7 +366,7 @@ class BaseS3StorageDriver(StorageDriver):
 :rtype: ``str``
 """
 container_url = self._get_container_path(container)
- object_name_cleaned = self._clean_object_name(object_name)
+ object_name_cleaned = object_name
 object_path = '%s/%s' % (container_url, object_name_cleaned)
 return object_path

@@ -776,9 +776,6 @@ class BaseS3StorageDriver(StorageDriver):
 delimiter=None):
 self._abort_multipart(container, upload.key, upload.id)

- def _clean_object_name(self, name):
- name = urlquote(name)
- return name

 def _put_object(self, container, object_name, method='PUT',
 query_args=None, extra=None, file_path=None,

~$ ./test.py 
<Object: name=file~, size=0, hash=d41d8cd98f00b204e9800998ecf8427e, provider=Ceph RGW S3 (default) ...>
{quote}
 

This kind of issues existed on other projects (see [https://github.com/aws/aws-sdk-php/issues/37] and [https://github.com/aws/aws-sdk-go/issues/45] for instance)

I do not know if this is specific to Ceph, or if this issue applies to amazon's S3 as well (regarding the issues above, this seems to be)

 

Thanks


> libcloud.storage / S3_RGW: cannot fetch some objects
> ----------------------------------------------------
>
>                 Key: LIBCLOUD-979
>                 URL: https://issues.apache.org/jira/browse/LIBCLOUD-979
>             Project: Libcloud
>          Issue Type: Bug
>          Components: Storage
>         Environment:  The issue have been encountered on libcloud 2.2.1, as well as trunk (last commit: aaf15742f4efac7947a2f47e6e140303dc8ec2d2)
>            Reporter: Jack
>            Priority: Major
>
> Libcloud cannot handle some legitimates objects
>  Here is an example, that will explain the issue
> Please check this code:
> {quote}from libcloud.storage.types import Provider
>  from libcloud.storage.providers import get_driver
> driver = get_driver(Provider.S3_RGW)(key='access_key', secret='secret_key', host='hostname')
> print(driver.get_object('tilde', 'file~'))
> {quote}
>  
> Output:
> {quote}~$ ./test.py 
>  Traceback (most recent call last):
>  File "./test.py", line 8, in <module>
>  print(driver.get_object('tilde', 'file~'))
>  File "/tmp/libcloud/libcloud/storage/drivers/s3.py", line 332, in get_object
>  response = self.connection.request(object_path, method='HEAD')
>  File "/tmp/libcloud/libcloud/common/base.py", line 637, in request
>  response = responseCls(**kwargs)
>  File "/tmp/libcloud/libcloud/common/base.py", line 152, in __init__
>  message=self.parse_error(),
>  File "/tmp/libcloud/libcloud/storage/drivers/s3.py", line 96, in parse_error
>  raise InvalidCredsError(self.body)
>  libcloud.common.types.InvalidCredsError: ''
> {quote}
>  
> A tcpdump shows that libcloud tranforms the key from "file~" to "file%7E" (thus, url-encode)
> The slight patch below fixes this issue:
> {quote}diff --git a/libcloud/storage/drivers/s3.py b/libcloud/storage/drivers/s3.py
>  index f9f4b6c9..7107a8c6 100644
>  — a/libcloud/storage/drivers/s3.py
>  +++ b/libcloud/storage/drivers/s3.py
>  @@ -366,7 +366,7 @@ class BaseS3StorageDriver(StorageDriver):
>  :rtype: ``str``
>  """
>  container_url = self._get_container_path(container)
>  - object_name_cleaned = self._clean_object_name(object_name)
>  + object_name_cleaned = object_name
>  object_path = '%s/%s' % (container_url, object_name_cleaned)
>  return object_path
> @@ -776,9 +776,6 @@ class BaseS3StorageDriver(StorageDriver):
>  delimiter=None):
>  self._abort_multipart(container, upload.key, upload.id)
>  - def _clean_object_name(self, name):
>  - name = urlquote(name)
>  - return name
> def _put_object(self, container, object_name, method='PUT',
>  query_args=None, extra=None, file_path=None,
> ~$ ./test.py 
>  <Object: name=file~, size=0, hash=d41d8cd98f00b204e9800998ecf8427e, provider=Ceph RGW S3 (default) ...>
> {quote}
>  
> This kind of issues existed on other projects (see [https://github.com/aws/aws-sdk-php/issues/37] and [https://github.com/aws/aws-sdk-go/issues/45] for instance)
> I do not know if this is specific to Ceph, or if this issue applies to amazon's S3 as well (regarding the issues above, this seems to be)
>  
> Thanks



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)