You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@libcloud.apache.org by to...@apache.org on 2014/08/19 18:24:03 UTC
[2/2] git commit: Add support for HTTP proxy to
LibcloudHTTPConnection and LibcloudHTTPSConnection class.
Add support for HTTP proxy to LibcloudHTTPConnection and
LibcloudHTTPSConnection class.
User can specify which HTTP proxy to use using one of the following
approaches:
* by setting "proxy_url" environment variable (global / process wide)
* by passing "proxy_url" argument to the Connection class constructor (per
connection instance)
* by calling "set_http_proxy" method on the Connection class (per connection
instance)
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo
Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/f3f60002
Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/f3f60002
Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/f3f60002
Branch: refs/heads/trunk
Commit: f3f600028384bc19c0f7ff051306a401d57afd35
Parents: 5207038
Author: Tomaz Muraus <to...@apache.org>
Authored: Tue Aug 19 16:29:54 2014 +0200
Committer: Tomaz Muraus <to...@apache.org>
Committed: Tue Aug 19 18:22:18 2014 +0200
----------------------------------------------------------------------
.../examples/http_proxy/constructor_argument.py | 6 +
.../http_proxy/set_http_proxy_method.py | 12 ++
docs/other/using-http-proxy.rst | 52 ++++++++
libcloud/common/base.py | 15 ++-
libcloud/httplib_ssl.py | 129 ++++++++++++++++++-
libcloud/test/test_connection.py | 53 ++++++++
6 files changed, 262 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/libcloud/blob/f3f60002/docs/examples/http_proxy/constructor_argument.py
----------------------------------------------------------------------
diff --git a/docs/examples/http_proxy/constructor_argument.py b/docs/examples/http_proxy/constructor_argument.py
new file mode 100644
index 0000000..ae54945
--- /dev/null
+++ b/docs/examples/http_proxy/constructor_argument.py
@@ -0,0 +1,6 @@
+from libcloud.compute.drivers.dreamhost import DreamhostConnection
+
+PROXY_URL = 'http://<proxy hostname>:<proxy port>'
+
+conn = DreamhostConnection(host='dreamhost.com', port=443,
+ timeout=None, proxy_url=PROXY_URL)
http://git-wip-us.apache.org/repos/asf/libcloud/blob/f3f60002/docs/examples/http_proxy/set_http_proxy_method.py
----------------------------------------------------------------------
diff --git a/docs/examples/http_proxy/set_http_proxy_method.py b/docs/examples/http_proxy/set_http_proxy_method.py
new file mode 100644
index 0000000..d8c5e83
--- /dev/null
+++ b/docs/examples/http_proxy/set_http_proxy_method.py
@@ -0,0 +1,12 @@
+from pprint import pprint
+
+from libcloud.compute.types import Provider
+from libcloud.compute.providers import get_driver
+
+PROXY_URL = 'http://<proxy hostname>:<proxy port>'
+
+cls = get_driver(Provider.RACKSPACE)
+driver = cls('username', 'api key', region='ord')
+driver.set_http_proxy(proxy_url=PROXY_URL)
+
+pprint(driver.list_nodes())
http://git-wip-us.apache.org/repos/asf/libcloud/blob/f3f60002/docs/other/using-http-proxy.rst
----------------------------------------------------------------------
diff --git a/docs/other/using-http-proxy.rst b/docs/other/using-http-proxy.rst
new file mode 100644
index 0000000..8c12dc5
--- /dev/null
+++ b/docs/other/using-http-proxy.rst
@@ -0,0 +1,52 @@
+Using an HTTP proxy
+===================
+
+.. note::
+
+ Support for HTTP proxies is only available in Libcloud trunk and higher.
+
+Libcloud supports using an HTTP proxy for outgoing HTTP and HTTPS requests. At
+the moment, using a proxy is only supported if you are using Python 2.7 or
+above (it has been tested with 2.7, PyPy, 3.1, 3.3, 3.3, 3.4).
+
+You can specify which HTTP proxy to use using one of the approaches described
+bellow:
+
+* By setting ``http_proxy`` environment variable (this setting is system /
+ process wide)
+* By passing ``http_proxy`` argument to the
+ :class:`libcloud.common.base.LibcloudHTTPConnection` class constructor (this
+ setting is local to the connection instance)
+* By calling :meth:`libcloud.common.base.LibcloudHTTPConnection.set_http_proxy`
+ method (this setting is local to the connection instance)
+
+Known limitations
+-----------------
+
+* HTTP proxies which require authentication are not supported
+* Python 2.6 is not supported
+
+Examples
+--------
+
+This section includes some code examples which show how to use an HTTP proxy
+with Libcloud.
+
+1. Using http_proxy environment variable
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. sourcecode:: python
+
+ http_proxy=http://<proxy hostname>:<proxy port> python my_script.py
+
+2. Passing http_proxy argument to the connection class
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. literalinclude:: /examples/http_proxy/constructor_argument.py
+ :language: python
+
+3. Calling set_http_proxy method
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. literalinclude:: /examples/http_proxy/set_http_proxy_method.py
+ :language: python
http://git-wip-us.apache.org/repos/asf/libcloud/blob/f3f60002/libcloud/common/base.py
----------------------------------------------------------------------
diff --git a/libcloud/common/base.py b/libcloud/common/base.py
index 488f084..daf04d2 100644
--- a/libcloud/common/base.py
+++ b/libcloud/common/base.py
@@ -46,10 +46,9 @@ from libcloud.utils.misc import lowercase_keys
from libcloud.utils.compression import decompress_data
from libcloud.common.types import LibcloudError, MalformedResponseError
+from libcloud.httplib_ssl import LibcloudHTTPConnection
from libcloud.httplib_ssl import LibcloudHTTPSConnection
-LibcloudHTTPConnection = httplib.HTTPConnection
-
class HTTPResponse(httplib.HTTPResponse):
# On python 2.6 some calls can hang because HEAD isn't quite properly
@@ -267,7 +266,9 @@ class LoggingConnection():
:cvar log: file-like object that logs entries are written to.
"""
+
log = None
+ http_proxy_used = False
def _log_response(self, r):
rv = "# -------- begin %d:%d response ----------\n" % (id(self), id(r))
@@ -341,7 +342,15 @@ class LoggingConnection():
return (rr, rv)
def _log_curl(self, method, url, body, headers):
- cmd = ["curl", "-i"]
+ cmd = ["curl"]
+
+ if self.http_proxy_used:
+ proxy_url = 'http://%s:%s' % (self.proxy_host,
+ self.proxy_port)
+ proxy_url = pquote(proxy_url)
+ cmd.extend(['--proxy', proxy_url])
+
+ cmd.extend(['-i'])
if method.lower() == 'head':
# HEAD method need special handling
http://git-wip-us.apache.org/repos/asf/libcloud/blob/f3f60002/libcloud/httplib_ssl.py
----------------------------------------------------------------------
diff --git a/libcloud/httplib_ssl.py b/libcloud/httplib_ssl.py
index 29136ef..2024433 100644
--- a/libcloud/httplib_ssl.py
+++ b/libcloud/httplib_ssl.py
@@ -19,14 +19,126 @@ verification, depending on libcloud.security settings.
import os
import re
import socket
+import sys
import ssl
import warnings
import libcloud.security
from libcloud.utils.py3 import httplib
+from libcloud.utils.py3 import urlparse
+__all__ = [
+ 'LibcloudBaseConnection',
+ 'LibcloudHTTPConnection',
+ 'LibcloudHTTPSConnection'
+]
-class LibcloudHTTPSConnection(httplib.HTTPSConnection):
+HTTP_PROXY_ENV_VARIABLE_NAME = 'http_proxy'
+
+
+class LibcloudBaseConnection(object):
+ """
+ Base connection class to inherit from.
+
+ Note: This class should not be instantiated directly.
+ """
+
+ proxy_scheme = None
+ proxy_host = None
+ proxy_port = None
+ http_proxy_used = False
+
+ def set_http_proxy(self, proxy_url):
+ """
+ Set a HTTP proxy which will be used with this connection.
+
+ :param proxy_url: Proxy URL (e.g. http://hostname:3128)
+ :type proxy_url: ``str``
+ """
+ if sys.version_info[:2] == (2, 6):
+ raise Exception('HTTP proxy support requires Python 2.7 or higher')
+
+ scheme, host, port = self._parse_proxy_url(proxy_url=proxy_url)
+
+ self.proxy_scheme = scheme
+ self.proxy_host = host
+ self.proxy_port = port
+
+ self._setup_http_proxy()
+
+ def _parse_proxy_url(self, proxy_url):
+ """
+ Parse and validate a proxy URL.
+
+ :param proxy_url: Proxy URL (e.g. http://hostname:3128)
+ :type proxy_url: ``str``
+
+ :rtype: ``tuple`` (``scheme``, ``hostname``, ``port``)
+ """
+ parsed = urlparse.urlparse(proxy_url)
+
+ if parsed.scheme != 'http':
+ raise ValueError('Only http proxies are supported')
+
+ if not parsed.hostname or not parsed.port:
+ raise ValueError('proxy_url must be in the following format: '
+ 'http://<proxy host>:<proxy port>')
+
+ proxy_scheme = parsed.scheme
+ proxy_host, proxy_port = parsed.hostname, parsed.port
+
+ return (proxy_scheme, proxy_host, proxy_port)
+
+ def _setup_http_proxy(self):
+ """
+ Set up HTTP proxy.
+
+ :param proxy_url: Proxy URL (e.g. http://<host>:3128)
+ :type proxy_url: ``str``
+ """
+ self.set_tunnel(host=self.host, port=self.port)
+ self._set_hostport(host=self.proxy_host, port=self.proxy_port)
+
+ def _activate_http_proxy(self, sock):
+ self.sock = sock
+ self._tunnel()
+
+ def _set_hostport(self, host, port):
+ """
+ Backported from Python stdlib so Proxy support also works with
+ Python 3.4.
+ """
+ if port is None:
+ i = host.rfind(':')
+ j = host.rfind(']') # ipv6 addresses have [...]
+ if i > j:
+ try:
+ port = int(host[i+1:])
+ except ValueError:
+ msg = "nonnumeric port: '%s'" % host[i+1:]
+ raise httplib.InvalidURL(msg)
+ host = host[:i]
+ else:
+ port = self.default_port
+ if host and host[0] == '[' and host[-1] == ']':
+ host = host[1:-1]
+ self.host = host
+ self.port = port
+
+
+class LibcloudHTTPConnection(httplib.HTTPConnection, LibcloudBaseConnection):
+ def __init__(self, *args, **kwargs):
+ # Support for HTTP proxy
+ proxy_url_env = os.environ.get(HTTP_PROXY_ENV_VARIABLE_NAME, None)
+ proxy_url = kwargs.pop('proxy_url', proxy_url_env)
+
+ super(LibcloudHTTPConnection, self).__init__(*args, **kwargs)
+
+ if proxy_url:
+ self.set_http_proxy(proxy_url=proxy_url)
+
+
+class LibcloudHTTPSConnection(httplib.HTTPSConnection, LibcloudBaseConnection):
"""
LibcloudHTTPSConnection
@@ -41,7 +153,15 @@ class LibcloudHTTPSConnection(httplib.HTTPSConnection):
Constructor
"""
self._setup_verify()
- httplib.HTTPSConnection.__init__(self, *args, **kwargs)
+
+ # Support for HTTP proxy
+ proxy_url_env = os.environ.get(HTTP_PROXY_ENV_VARIABLE_NAME, None)
+ proxy_url = kwargs.pop('proxy_url', proxy_url_env)
+
+ super(LibcloudHTTPSConnection, self).__init__(*args, **kwargs)
+
+ if proxy_url:
+ self.set_http_proxy(proxy_url=proxy_url)
def _setup_verify(self):
"""
@@ -97,6 +217,11 @@ class LibcloudHTTPSConnection(httplib.HTTPSConnection):
else:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((self.host, self.port))
+
+ # Activate the HTTP proxy
+ if self.http_proxy_used:
+ self._activate_http_proxy(sock=sock)
+
self.sock = ssl.wrap_socket(sock,
self.key_file,
self.cert_file,
http://git-wip-us.apache.org/repos/asf/libcloud/blob/f3f60002/libcloud/test/test_connection.py
----------------------------------------------------------------------
diff --git a/libcloud/test/test_connection.py b/libcloud/test/test_connection.py
index 5df7919..b5c2abb 100644
--- a/libcloud/test/test_connection.py
+++ b/libcloud/test/test_connection.py
@@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import os
import sys
import ssl
@@ -22,6 +23,58 @@ from mock import Mock, call
from libcloud.test import unittest
from libcloud.common.base import Connection
from libcloud.common.base import LoggingConnection
+from libcloud.httplib_ssl import LibcloudBaseConnection
+from libcloud.httplib_ssl import LibcloudHTTPConnection
+
+
+class BaseConnectionClassTestCase(unittest.TestCase):
+ def test_parse_proxy_url(self):
+ conn = LibcloudBaseConnection()
+
+ proxy_url = 'http://127.0.0.1:3128'
+ result = conn._parse_proxy_url(proxy_url=proxy_url)
+ self.assertEqual(result[0], 'http')
+ self.assertEqual(result[1], '127.0.0.1')
+ self.assertEqual(result[2], 3128)
+
+ proxy_url = 'https://127.0.0.1:3128'
+ expected_msg = 'Only http proxies are supported'
+ self.assertRaisesRegexp(ValueError, expected_msg,
+ conn._parse_proxy_url,
+ proxy_url=proxy_url)
+
+ proxy_url = 'http://127.0.0.1'
+ expected_msg = 'proxy_url must be in the following format'
+ self.assertRaisesRegexp(ValueError, expected_msg,
+ conn._parse_proxy_url,
+ proxy_url=proxy_url)
+
+ def test_constructor(self):
+ conn = LibcloudHTTPConnection(host='localhost', port=80)
+ self.assertEqual(conn.proxy_scheme, None)
+ self.assertEqual(conn.proxy_host, None)
+ self.assertEqual(conn.proxy_port, None)
+
+ proxy_url = 'http://127.0.0.3:3128'
+ conn.set_http_proxy(proxy_url=proxy_url)
+ self.assertEqual(conn.proxy_scheme, 'http')
+ self.assertEqual(conn.proxy_host, '127.0.0.3')
+ self.assertEqual(conn.proxy_port, 3128)
+
+ proxy_url = 'http://127.0.0.4:3128'
+ conn = LibcloudHTTPConnection(host='localhost', port=80,
+ proxy_url=proxy_url)
+ self.assertEqual(conn.proxy_scheme, 'http')
+ self.assertEqual(conn.proxy_host, '127.0.0.4')
+ self.assertEqual(conn.proxy_port, 3128)
+
+ os.environ['http_proxy'] = proxy_url
+ proxy_url = 'http://127.0.0.5:3128'
+ conn = LibcloudHTTPConnection(host='localhost', port=80,
+ proxy_url=proxy_url)
+ self.assertEqual(conn.proxy_scheme, 'http')
+ self.assertEqual(conn.proxy_host, '127.0.0.5')
+ self.assertEqual(conn.proxy_port, 3128)
class ConnectionClassTestCase(unittest.TestCase):