You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by li...@apache.org on 2017/01/16 04:56:38 UTC
[1/5] incubator-trafodion git commit: [TRAFODION-2431] python
installer - add python lib dependencies
Repository: incubator-trafodion
Updated Branches:
refs/heads/master c572f6389 -> 4261a9d66
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/702b6ddd/install/python-installer/scripts/httplib2/socks.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/httplib2/socks.py b/install/python-installer/scripts/httplib2/socks.py
new file mode 100644
index 0000000..0991f4c
--- /dev/null
+++ b/install/python-installer/scripts/httplib2/socks.py
@@ -0,0 +1,438 @@
+"""SocksiPy - Python SOCKS module.
+Version 1.00
+
+Copyright 2006 Dan-Haim. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+1. Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+3. Neither the name of Dan Haim nor the names of his contributors may be used
+ to endorse or promote products derived from this software without specific
+ prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY DAN HAIM "AS IS" AND ANY EXPRESS OR IMPLIED
+WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+EVENT SHALL DAN HAIM OR HIS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA
+OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMANGE.
+
+
+This module provides a standard socket-like interface for Python
+for tunneling connections through SOCKS proxies.
+
+"""
+
+"""
+
+Minor modifications made by Christopher Gilbert (http://motomastyle.com/)
+for use in PyLoris (http://pyloris.sourceforge.net/)
+
+Minor modifications made by Mario Vilas (http://breakingcode.wordpress.com/)
+mainly to merge bug fixes found in Sourceforge
+
+"""
+
+import base64
+import socket
+import struct
+import sys
+
+if getattr(socket, 'socket', None) is None:
+ raise ImportError('socket.socket missing, proxy support unusable')
+
+PROXY_TYPE_SOCKS4 = 1
+PROXY_TYPE_SOCKS5 = 2
+PROXY_TYPE_HTTP = 3
+PROXY_TYPE_HTTP_NO_TUNNEL = 4
+
+_defaultproxy = None
+_orgsocket = socket.socket
+
+class ProxyError(Exception): pass
+class GeneralProxyError(ProxyError): pass
+class Socks5AuthError(ProxyError): pass
+class Socks5Error(ProxyError): pass
+class Socks4Error(ProxyError): pass
+class HTTPError(ProxyError): pass
+
+_generalerrors = ("success",
+ "invalid data",
+ "not connected",
+ "not available",
+ "bad proxy type",
+ "bad input")
+
+_socks5errors = ("succeeded",
+ "general SOCKS server failure",
+ "connection not allowed by ruleset",
+ "Network unreachable",
+ "Host unreachable",
+ "Connection refused",
+ "TTL expired",
+ "Command not supported",
+ "Address type not supported",
+ "Unknown error")
+
+_socks5autherrors = ("succeeded",
+ "authentication is required",
+ "all offered authentication methods were rejected",
+ "unknown username or invalid password",
+ "unknown error")
+
+_socks4errors = ("request granted",
+ "request rejected or failed",
+ "request rejected because SOCKS server cannot connect to identd on the client",
+ "request rejected because the client program and identd report different user-ids",
+ "unknown error")
+
+def setdefaultproxy(proxytype=None, addr=None, port=None, rdns=True, username=None, password=None):
+ """setdefaultproxy(proxytype, addr[, port[, rdns[, username[, password]]]])
+ Sets a default proxy which all further socksocket objects will use,
+ unless explicitly changed.
+ """
+ global _defaultproxy
+ _defaultproxy = (proxytype, addr, port, rdns, username, password)
+
+def wrapmodule(module):
+ """wrapmodule(module)
+ Attempts to replace a module's socket library with a SOCKS socket. Must set
+ a default proxy using setdefaultproxy(...) first.
+ This will only work on modules that import socket directly into the namespace;
+ most of the Python Standard Library falls into this category.
+ """
+ if _defaultproxy != None:
+ module.socket.socket = socksocket
+ else:
+ raise GeneralProxyError((4, "no proxy specified"))
+
+class socksocket(socket.socket):
+ """socksocket([family[, type[, proto]]]) -> socket object
+ Open a SOCKS enabled socket. The parameters are the same as
+ those of the standard socket init. In order for SOCKS to work,
+ you must specify family=AF_INET, type=SOCK_STREAM and proto=0.
+ """
+
+ def __init__(self, family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0, _sock=None):
+ _orgsocket.__init__(self, family, type, proto, _sock)
+ if _defaultproxy != None:
+ self.__proxy = _defaultproxy
+ else:
+ self.__proxy = (None, None, None, None, None, None)
+ self.__proxysockname = None
+ self.__proxypeername = None
+ self.__httptunnel = True
+
+ def __recvall(self, count):
+ """__recvall(count) -> data
+ Receive EXACTLY the number of bytes requested from the socket.
+ Blocks until the required number of bytes have been received.
+ """
+ data = self.recv(count)
+ while len(data) < count:
+ d = self.recv(count-len(data))
+ if not d: raise GeneralProxyError((0, "connection closed unexpectedly"))
+ data = data + d
+ return data
+
+ def sendall(self, content, *args):
+ """ override socket.socket.sendall method to rewrite the header
+ for non-tunneling proxies if needed
+ """
+ if not self.__httptunnel:
+ content = self.__rewriteproxy(content)
+ return super(socksocket, self).sendall(content, *args)
+
+ def __rewriteproxy(self, header):
+ """ rewrite HTTP request headers to support non-tunneling proxies
+ (i.e. those which do not support the CONNECT method).
+ This only works for HTTP (not HTTPS) since HTTPS requires tunneling.
+ """
+ host, endpt = None, None
+ hdrs = header.split("\r\n")
+ for hdr in hdrs:
+ if hdr.lower().startswith("host:"):
+ host = hdr
+ elif hdr.lower().startswith("get") or hdr.lower().startswith("post"):
+ endpt = hdr
+ if host and endpt:
+ hdrs.remove(host)
+ hdrs.remove(endpt)
+ host = host.split(" ")[1]
+ endpt = endpt.split(" ")
+ if (self.__proxy[4] != None and self.__proxy[5] != None):
+ hdrs.insert(0, self.__getauthheader())
+ hdrs.insert(0, "Host: %s" % host)
+ hdrs.insert(0, "%s http://%s%s %s" % (endpt[0], host, endpt[1], endpt[2]))
+ return "\r\n".join(hdrs)
+
+ def __getauthheader(self):
+ auth = self.__proxy[4] + ":" + self.__proxy[5]
+ return "Proxy-Authorization: Basic " + base64.b64encode(auth)
+
+ def setproxy(self, proxytype=None, addr=None, port=None, rdns=True, username=None, password=None):
+ """setproxy(proxytype, addr[, port[, rdns[, username[, password]]]])
+ Sets the proxy to be used.
+ proxytype - The type of the proxy to be used. Three types
+ are supported: PROXY_TYPE_SOCKS4 (including socks4a),
+ PROXY_TYPE_SOCKS5 and PROXY_TYPE_HTTP
+ addr - The address of the server (IP or DNS).
+ port - The port of the server. Defaults to 1080 for SOCKS
+ servers and 8080 for HTTP proxy servers.
+ rdns - Should DNS queries be preformed on the remote side
+ (rather than the local side). The default is True.
+ Note: This has no effect with SOCKS4 servers.
+ username - Username to authenticate with to the server.
+ The default is no authentication.
+ password - Password to authenticate with to the server.
+ Only relevant when username is also provided.
+ """
+ self.__proxy = (proxytype, addr, port, rdns, username, password)
+
+ def __negotiatesocks5(self, destaddr, destport):
+ """__negotiatesocks5(self,destaddr,destport)
+ Negotiates a connection through a SOCKS5 server.
+ """
+ # First we'll send the authentication packages we support.
+ if (self.__proxy[4]!=None) and (self.__proxy[5]!=None):
+ # The username/password details were supplied to the
+ # setproxy method so we support the USERNAME/PASSWORD
+ # authentication (in addition to the standard none).
+ self.sendall(struct.pack('BBBB', 0x05, 0x02, 0x00, 0x02))
+ else:
+ # No username/password were entered, therefore we
+ # only support connections with no authentication.
+ self.sendall(struct.pack('BBB', 0x05, 0x01, 0x00))
+ # We'll receive the server's response to determine which
+ # method was selected
+ chosenauth = self.__recvall(2)
+ if chosenauth[0:1] != chr(0x05).encode():
+ self.close()
+ raise GeneralProxyError((1, _generalerrors[1]))
+ # Check the chosen authentication method
+ if chosenauth[1:2] == chr(0x00).encode():
+ # No authentication is required
+ pass
+ elif chosenauth[1:2] == chr(0x02).encode():
+ # Okay, we need to perform a basic username/password
+ # authentication.
+ self.sendall(chr(0x01).encode() + chr(len(self.__proxy[4])) + self.__proxy[4] + chr(len(self.__proxy[5])) + self.__proxy[5])
+ authstat = self.__recvall(2)
+ if authstat[0:1] != chr(0x01).encode():
+ # Bad response
+ self.close()
+ raise GeneralProxyError((1, _generalerrors[1]))
+ if authstat[1:2] != chr(0x00).encode():
+ # Authentication failed
+ self.close()
+ raise Socks5AuthError((3, _socks5autherrors[3]))
+ # Authentication succeeded
+ else:
+ # Reaching here is always bad
+ self.close()
+ if chosenauth[1] == chr(0xFF).encode():
+ raise Socks5AuthError((2, _socks5autherrors[2]))
+ else:
+ raise GeneralProxyError((1, _generalerrors[1]))
+ # Now we can request the actual connection
+ req = struct.pack('BBB', 0x05, 0x01, 0x00)
+ # If the given destination address is an IP address, we'll
+ # use the IPv4 address request even if remote resolving was specified.
+ try:
+ ipaddr = socket.inet_aton(destaddr)
+ req = req + chr(0x01).encode() + ipaddr
+ except socket.error:
+ # Well it's not an IP number, so it's probably a DNS name.
+ if self.__proxy[3]:
+ # Resolve remotely
+ ipaddr = None
+ req = req + chr(0x03).encode() + chr(len(destaddr)).encode() + destaddr
+ else:
+ # Resolve locally
+ ipaddr = socket.inet_aton(socket.gethostbyname(destaddr))
+ req = req + chr(0x01).encode() + ipaddr
+ req = req + struct.pack(">H", destport)
+ self.sendall(req)
+ # Get the response
+ resp = self.__recvall(4)
+ if resp[0:1] != chr(0x05).encode():
+ self.close()
+ raise GeneralProxyError((1, _generalerrors[1]))
+ elif resp[1:2] != chr(0x00).encode():
+ # Connection failed
+ self.close()
+ if ord(resp[1:2])<=8:
+ raise Socks5Error((ord(resp[1:2]), _socks5errors[ord(resp[1:2])]))
+ else:
+ raise Socks5Error((9, _socks5errors[9]))
+ # Get the bound address/port
+ elif resp[3:4] == chr(0x01).encode():
+ boundaddr = self.__recvall(4)
+ elif resp[3:4] == chr(0x03).encode():
+ resp = resp + self.recv(1)
+ boundaddr = self.__recvall(ord(resp[4:5]))
+ else:
+ self.close()
+ raise GeneralProxyError((1,_generalerrors[1]))
+ boundport = struct.unpack(">H", self.__recvall(2))[0]
+ self.__proxysockname = (boundaddr, boundport)
+ if ipaddr != None:
+ self.__proxypeername = (socket.inet_ntoa(ipaddr), destport)
+ else:
+ self.__proxypeername = (destaddr, destport)
+
+ def getproxysockname(self):
+ """getsockname() -> address info
+ Returns the bound IP address and port number at the proxy.
+ """
+ return self.__proxysockname
+
+ def getproxypeername(self):
+ """getproxypeername() -> address info
+ Returns the IP and port number of the proxy.
+ """
+ return _orgsocket.getpeername(self)
+
+ def getpeername(self):
+ """getpeername() -> address info
+ Returns the IP address and port number of the destination
+ machine (note: getproxypeername returns the proxy)
+ """
+ return self.__proxypeername
+
+ def __negotiatesocks4(self,destaddr,destport):
+ """__negotiatesocks4(self,destaddr,destport)
+ Negotiates a connection through a SOCKS4 server.
+ """
+ # Check if the destination address provided is an IP address
+ rmtrslv = False
+ try:
+ ipaddr = socket.inet_aton(destaddr)
+ except socket.error:
+ # It's a DNS name. Check where it should be resolved.
+ if self.__proxy[3]:
+ ipaddr = struct.pack("BBBB", 0x00, 0x00, 0x00, 0x01)
+ rmtrslv = True
+ else:
+ ipaddr = socket.inet_aton(socket.gethostbyname(destaddr))
+ # Construct the request packet
+ req = struct.pack(">BBH", 0x04, 0x01, destport) + ipaddr
+ # The username parameter is considered userid for SOCKS4
+ if self.__proxy[4] != None:
+ req = req + self.__proxy[4]
+ req = req + chr(0x00).encode()
+ # DNS name if remote resolving is required
+ # NOTE: This is actually an extension to the SOCKS4 protocol
+ # called SOCKS4A and may not be supported in all cases.
+ if rmtrslv:
+ req = req + destaddr + chr(0x00).encode()
+ self.sendall(req)
+ # Get the response from the server
+ resp = self.__recvall(8)
+ if resp[0:1] != chr(0x00).encode():
+ # Bad data
+ self.close()
+ raise GeneralProxyError((1,_generalerrors[1]))
+ if resp[1:2] != chr(0x5A).encode():
+ # Server returned an error
+ self.close()
+ if ord(resp[1:2]) in (91, 92, 93):
+ self.close()
+ raise Socks4Error((ord(resp[1:2]), _socks4errors[ord(resp[1:2]) - 90]))
+ else:
+ raise Socks4Error((94, _socks4errors[4]))
+ # Get the bound address/port
+ self.__proxysockname = (socket.inet_ntoa(resp[4:]), struct.unpack(">H", resp[2:4])[0])
+ if rmtrslv != None:
+ self.__proxypeername = (socket.inet_ntoa(ipaddr), destport)
+ else:
+ self.__proxypeername = (destaddr, destport)
+
+ def __negotiatehttp(self, destaddr, destport):
+ """__negotiatehttp(self,destaddr,destport)
+ Negotiates a connection through an HTTP server.
+ """
+ # If we need to resolve locally, we do this now
+ if not self.__proxy[3]:
+ addr = socket.gethostbyname(destaddr)
+ else:
+ addr = destaddr
+ headers = ["CONNECT ", addr, ":", str(destport), " HTTP/1.1\r\n"]
+ headers += ["Host: ", destaddr, "\r\n"]
+ if (self.__proxy[4] != None and self.__proxy[5] != None):
+ headers += [self.__getauthheader(), "\r\n"]
+ headers.append("\r\n")
+ self.sendall("".join(headers).encode())
+ # We read the response until we get the string "\r\n\r\n"
+ resp = self.recv(1)
+ while resp.find("\r\n\r\n".encode()) == -1:
+ resp = resp + self.recv(1)
+ # We just need the first line to check if the connection
+ # was successful
+ statusline = resp.splitlines()[0].split(" ".encode(), 2)
+ if statusline[0] not in ("HTTP/1.0".encode(), "HTTP/1.1".encode()):
+ self.close()
+ raise GeneralProxyError((1, _generalerrors[1]))
+ try:
+ statuscode = int(statusline[1])
+ except ValueError:
+ self.close()
+ raise GeneralProxyError((1, _generalerrors[1]))
+ if statuscode != 200:
+ self.close()
+ raise HTTPError((statuscode, statusline[2]))
+ self.__proxysockname = ("0.0.0.0", 0)
+ self.__proxypeername = (addr, destport)
+
+ def connect(self, destpair):
+ """connect(self, despair)
+ Connects to the specified destination through a proxy.
+ destpar - A tuple of the IP/DNS address and the port number.
+ (identical to socket's connect).
+ To select the proxy server use setproxy().
+ """
+ # Do a minimal input check first
+ if (not type(destpair) in (list,tuple)) or (len(destpair) < 2) or (not isinstance(destpair[0], basestring)) or (type(destpair[1]) != int):
+ raise GeneralProxyError((5, _generalerrors[5]))
+ if self.__proxy[0] == PROXY_TYPE_SOCKS5:
+ if self.__proxy[2] != None:
+ portnum = self.__proxy[2]
+ else:
+ portnum = 1080
+ _orgsocket.connect(self, (self.__proxy[1], portnum))
+ self.__negotiatesocks5(destpair[0], destpair[1])
+ elif self.__proxy[0] == PROXY_TYPE_SOCKS4:
+ if self.__proxy[2] != None:
+ portnum = self.__proxy[2]
+ else:
+ portnum = 1080
+ _orgsocket.connect(self,(self.__proxy[1], portnum))
+ self.__negotiatesocks4(destpair[0], destpair[1])
+ elif self.__proxy[0] == PROXY_TYPE_HTTP:
+ if self.__proxy[2] != None:
+ portnum = self.__proxy[2]
+ else:
+ portnum = 8080
+ _orgsocket.connect(self,(self.__proxy[1], portnum))
+ self.__negotiatehttp(destpair[0], destpair[1])
+ elif self.__proxy[0] == PROXY_TYPE_HTTP_NO_TUNNEL:
+ if self.__proxy[2] != None:
+ portnum = self.__proxy[2]
+ else:
+ portnum = 8080
+ _orgsocket.connect(self,(self.__proxy[1],portnum))
+ if destpair[1] == 443:
+ self.__negotiatehttp(destpair[0],destpair[1])
+ else:
+ self.__httptunnel = False
+ elif self.__proxy[0] == None:
+ _orgsocket.connect(self, (destpair[0], destpair[1]))
+ else:
+ raise GeneralProxyError((4, _generalerrors[4]))
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/702b6ddd/licenses/Makefile
----------------------------------------------------------------------
diff --git a/licenses/Makefile b/licenses/Makefile
index ec3198c..cc194b7 100644
--- a/licenses/Makefile
+++ b/licenses/Makefile
@@ -19,11 +19,11 @@
#
# @@@ END COPYRIGHT @@@
-all: LICENSE-src LICENSE-server LICENSE-install LICENSE-clients NOTICE-server DISCLAIMER-server
+all: LICENSE-src LICENSE-server LICENSE-pyinstall LICENSE-install LICENSE-clients NOTICE-server DISCLAIMER-server
# All source code included in Trafodion source
LICENSE-src:
- cat Apache lic-components-src lic-server-src lic-test-src > $@
+ cat Apache lic-components-src lic-server-src lic-test-src lic-pyinstall-src > $@
# Binary licenses for each separately packaged component
# Must include source licenses as well as software bundled in at build time
@@ -39,6 +39,9 @@ DISCLAIMER-server:
LICENSE-install:
cat Apache > $@
+LICENSE-pyinstall:
+ cat Apache lic-pyinstall-src > $@
+
LICENSE-clients:
cat Apache lic-clients-bin > $@
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/702b6ddd/licenses/lic-pyinstall-src
----------------------------------------------------------------------
diff --git a/licenses/lic-pyinstall-src b/licenses/lic-pyinstall-src
new file mode 100644
index 0000000..1064450
--- /dev/null
+++ b/licenses/lic-pyinstall-src
@@ -0,0 +1,63 @@
+===========================================================================
+The python installer dependencies bundles source with BSD-like and
+MIT-like licenses:
+
++++++++++++++++++++++++++++++
+
+BSD-3 clause for files in:
+ incubator-trafodion/installer/python-installer/prettytable.py
+ incubator-trafodion/installer/python-installer/scripts/httplib2/socks.py
+
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+* Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+* The name of the author may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
++++++++++++++++++++++++++++++
+
+MIT-Expat for files in:
+ incubator-trafodion/installer/python-installer/scripts/httplib2/iri2uri.py
+ incubator-trafodion/installer/python-installer/scripts/httplib2/__init__.py
+
+Httplib2 Software License
+
+Copyright (c) 2006 by Joe Gregorio
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without restriction,
+including without limitation the rights to use, copy, modify, merge,
+publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
[4/5] incubator-trafodion git commit: update .rat-excludes for
pyinstaller dependencies
Posted by li...@apache.org.
update .rat-excludes for pyinstaller dependencies
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/28c98a64
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/28c98a64
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/28c98a64
Branch: refs/heads/master
Commit: 28c98a64def20df9d6516b92aea65e57c9609978
Parents: 702b6dd
Author: Eason <hf...@gmail.com>
Authored: Tue Jan 10 16:01:10 2017 +0800
Committer: Eason <hf...@gmail.com>
Committed: Wed Jan 11 21:27:06 2017 +0800
----------------------------------------------------------------------
.rat-excludes | 6 +++++-
licenses/lic-pyinstall-src | 8 ++++----
2 files changed, 9 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/28c98a64/.rat-excludes
----------------------------------------------------------------------
diff --git a/.rat-excludes b/.rat-excludes
index ff8b481..e24f78d 100644
--- a/.rat-excludes
+++ b/.rat-excludes
@@ -166,7 +166,7 @@ VariableLengthPKTest.java
# asciidoc extention file -- MIT license
google-analytics-postprocessor.rb
-# installer JSON config files
+# pyinstaller JSON config files
mod_cfgs.json
prompt.json
script.json
@@ -175,3 +175,7 @@ version.json
mpack.json
role_command_order.json
theme.json
+# pyinstaller dependencies
+# they have BSD-3 license that RAT does not recognize
+prettytable.py
+socks.py
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/28c98a64/licenses/lic-pyinstall-src
----------------------------------------------------------------------
diff --git a/licenses/lic-pyinstall-src b/licenses/lic-pyinstall-src
index 1064450..f0f74c2 100644
--- a/licenses/lic-pyinstall-src
+++ b/licenses/lic-pyinstall-src
@@ -5,8 +5,8 @@ MIT-like licenses:
+++++++++++++++++++++++++++++
BSD-3 clause for files in:
- incubator-trafodion/installer/python-installer/prettytable.py
- incubator-trafodion/installer/python-installer/scripts/httplib2/socks.py
+ installer/python-installer/prettytable.py
+ installer/python-installer/scripts/httplib2/socks.py
Redistribution and use in source and binary forms, with or without
@@ -35,8 +35,8 @@ POSSIBILITY OF SUCH DAMAGE.
+++++++++++++++++++++++++++++
MIT-Expat for files in:
- incubator-trafodion/installer/python-installer/scripts/httplib2/iri2uri.py
- incubator-trafodion/installer/python-installer/scripts/httplib2/__init__.py
+ installer/python-installer/scripts/httplib2/iri2uri.py
+ installer/python-installer/scripts/httplib2/__init__.py
Httplib2 Software License
[5/5] incubator-trafodion git commit: Merge Fix for [TRAFODION-2431]
python installer - add python lib dependencies
Posted by li...@apache.org.
Merge Fix for [TRAFODION-2431] python installer - add python lib dependencies
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/4261a9d6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/4261a9d6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/4261a9d6
Branch: refs/heads/master
Commit: 4261a9d662e781d0138f456f5b2a35c113ca3e00
Parents: c572f63 28c98a6
Author: Liu Ming <ov...@sina.com>
Authored: Mon Jan 16 04:56:11 2017 +0000
Committer: Liu Ming <ov...@sina.com>
Committed: Mon Jan 16 04:56:11 2017 +0000
----------------------------------------------------------------------
.rat-excludes | 6 +-
install/Makefile | 5 +-
install/python-installer/prettytable.py | 1475 +++++++++++++++
.../scripts/httplib2/__init__.py | 1714 ++++++++++++++++++
.../scripts/httplib2/iri2uri.py | 132 ++
.../python-installer/scripts/httplib2/socks.py | 438 +++++
licenses/Makefile | 7 +-
licenses/lic-pyinstall-src | 63 +
8 files changed, 3836 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
[2/5] incubator-trafodion git commit: [TRAFODION-2431] python
installer - add python lib dependencies
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/702b6ddd/install/python-installer/scripts/httplib2/__init__.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/httplib2/__init__.py b/install/python-installer/scripts/httplib2/__init__.py
new file mode 100644
index 0000000..a15ca6b
--- /dev/null
+++ b/install/python-installer/scripts/httplib2/__init__.py
@@ -0,0 +1,1714 @@
+from __future__ import generators
+"""
+httplib2
+
+Copyright (c) 2006 by Joe Gregorio
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without restriction,
+including without limitation the rights to use, copy, modify, merge,
+publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+A caching http interface that supports ETags and gzip
+to conserve bandwidth.
+
+Requires Python 2.3 or later
+
+Changelog:
+2007-08-18, Rick: Modified so it's able to use a socks proxy if needed.
+
+"""
+
+__author__ = "Joe Gregorio (joe@bitworking.org)"
+__copyright__ = "Copyright 2006, Joe Gregorio"
+__contributors__ = ["Thomas Broyer (t.broyer@ltgt.net)",
+ "James Antill",
+ "Xavier Verges Farrero",
+ "Jonathan Feinberg",
+ "Blair Zajac",
+ "Sam Ruby",
+ "Louis Nyffenegger"]
+__license__ = "MIT"
+__version__ = "0.7.7"
+
+import re
+import sys
+import email
+import email.Utils
+import email.Message
+import email.FeedParser
+import StringIO
+import gzip
+import zlib
+import httplib
+import urlparse
+import urllib
+import base64
+import os
+import copy
+import calendar
+import time
+import random
+import errno
+try:
+ from hashlib import sha1 as _sha, md5 as _md5
+except ImportError:
+ # prior to Python 2.5, these were separate modules
+ import sha
+ import md5
+ _sha = sha.new
+ _md5 = md5.new
+import hmac
+from gettext import gettext as _
+import socket
+
+try:
+ from httplib2 import socks
+except ImportError:
+ try:
+ import socks
+ except ImportError:
+ socks = None
+
+# Build the appropriate socket wrapper for ssl
+try:
+ import ssl # python 2.6
+ ssl_SSLError = ssl.SSLError
+ def _ssl_wrap_socket(sock, key_file, cert_file,
+ disable_validation, ca_certs):
+ if disable_validation:
+ cert_reqs = ssl.CERT_NONE
+ else:
+ cert_reqs = ssl.CERT_REQUIRED
+ # We should be specifying SSL version 3 or TLS v1, but the ssl module
+ # doesn't expose the necessary knobs. So we need to go with the default
+ # of SSLv23.
+ return ssl.wrap_socket(sock, keyfile=key_file, certfile=cert_file,
+ cert_reqs=cert_reqs, ca_certs=ca_certs)
+except (AttributeError, ImportError):
+ ssl_SSLError = None
+ def _ssl_wrap_socket(sock, key_file, cert_file,
+ disable_validation, ca_certs):
+ if not disable_validation:
+ raise CertificateValidationUnsupported(
+ "SSL certificate validation is not supported without "
+ "the ssl module installed. To avoid this error, install "
+ "the ssl module, or explicity disable validation.")
+ ssl_sock = socket.ssl(sock, key_file, cert_file)
+ return httplib.FakeSocket(sock, ssl_sock)
+
+
+if sys.version_info >= (2,3):
+ from iri2uri import iri2uri
+else:
+ def iri2uri(uri):
+ return uri
+
+def has_timeout(timeout): # python 2.6
+ if hasattr(socket, '_GLOBAL_DEFAULT_TIMEOUT'):
+ return (timeout is not None and timeout is not socket._GLOBAL_DEFAULT_TIMEOUT)
+ return (timeout is not None)
+
+__all__ = ['Http', 'Response', 'ProxyInfo', 'HttpLib2Error',
+ 'RedirectMissingLocation', 'RedirectLimit', 'FailedToDecompressContent',
+ 'UnimplementedDigestAuthOptionError', 'UnimplementedHmacDigestAuthOptionError',
+ 'debuglevel', 'ProxiesUnavailableError']
+
+
+# The httplib debug level, set to a non-zero value to get debug output
+debuglevel = 0
+
+# A request will be tried 'RETRIES' times if it fails at the socket/connection level.
+RETRIES = 2
+
+# Python 2.3 support
+if sys.version_info < (2,4):
+ def sorted(seq):
+ seq.sort()
+ return seq
+
+# Python 2.3 support
+def HTTPResponse__getheaders(self):
+ """Return list of (header, value) tuples."""
+ if self.msg is None:
+ raise httplib.ResponseNotReady()
+ return self.msg.items()
+
+if not hasattr(httplib.HTTPResponse, 'getheaders'):
+ httplib.HTTPResponse.getheaders = HTTPResponse__getheaders
+
+# All exceptions raised here derive from HttpLib2Error
+class HttpLib2Error(Exception): pass
+
+# Some exceptions can be caught and optionally
+# be turned back into responses.
+class HttpLib2ErrorWithResponse(HttpLib2Error):
+ def __init__(self, desc, response, content):
+ self.response = response
+ self.content = content
+ HttpLib2Error.__init__(self, desc)
+
+class RedirectMissingLocation(HttpLib2ErrorWithResponse): pass
+class RedirectLimit(HttpLib2ErrorWithResponse): pass
+class FailedToDecompressContent(HttpLib2ErrorWithResponse): pass
+class UnimplementedDigestAuthOptionError(HttpLib2ErrorWithResponse): pass
+class UnimplementedHmacDigestAuthOptionError(HttpLib2ErrorWithResponse): pass
+
+class MalformedHeader(HttpLib2Error): pass
+class RelativeURIError(HttpLib2Error): pass
+class ServerNotFoundError(HttpLib2Error): pass
+class ProxiesUnavailableError(HttpLib2Error): pass
+class CertificateValidationUnsupported(HttpLib2Error): pass
+class SSLHandshakeError(HttpLib2Error): pass
+class NotSupportedOnThisPlatform(HttpLib2Error): pass
+class CertificateHostnameMismatch(SSLHandshakeError):
+ def __init__(self, desc, host, cert):
+ HttpLib2Error.__init__(self, desc)
+ self.host = host
+ self.cert = cert
+
+# Open Items:
+# -----------
+# Proxy support
+
+# Are we removing the cached content too soon on PUT (only delete on 200 Maybe?)
+
+# Pluggable cache storage (supports storing the cache in
+# flat files by default. We need a plug-in architecture
+# that can support Berkeley DB and Squid)
+
+# == Known Issues ==
+# Does not handle a resource that uses conneg and Last-Modified but no ETag as a cache validator.
+# Does not handle Cache-Control: max-stale
+# Does not use Age: headers when calculating cache freshness.
+
+
+# The number of redirections to follow before giving up.
+# Note that only GET redirects are automatically followed.
+# Will also honor 301 requests by saving that info and never
+# requesting that URI again.
+DEFAULT_MAX_REDIRECTS = 5
+
+# Default CA certificates file bundled with httplib2.
+#CA_CERTS = os.path.join(
+# os.path.dirname(os.path.abspath(__file__ )), "cacerts.txt")
+# We use fedora ca bundle file instead, to preserve backward compability.
+CA_CERTS = "/etc/ssl/certs/ca-bundle.crt"
+
+# Which headers are hop-by-hop headers by default
+HOP_BY_HOP = ['connection', 'keep-alive', 'proxy-authenticate', 'proxy-authorization', 'te', 'trailers', 'transfer-encoding', 'upgrade']
+
+def _get_end2end_headers(response):
+ hopbyhop = list(HOP_BY_HOP)
+ hopbyhop.extend([x.strip() for x in response.get('connection', '').split(',')])
+ return [header for header in response.keys() if header not in hopbyhop]
+
+URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?")
+
+def parse_uri(uri):
+ """Parses a URI using the regex given in Appendix B of RFC 3986.
+
+ (scheme, authority, path, query, fragment) = parse_uri(uri)
+ """
+ groups = URI.match(uri).groups()
+ return (groups[1], groups[3], groups[4], groups[6], groups[8])
+
+def urlnorm(uri):
+ (scheme, authority, path, query, fragment) = parse_uri(uri)
+ if not scheme or not authority:
+ raise RelativeURIError("Only absolute URIs are allowed. uri = %s" % uri)
+ authority = authority.lower()
+ scheme = scheme.lower()
+ if not path:
+ path = "/"
+ # Could do syntax based normalization of the URI before
+ # computing the digest. See Section 6.2.2 of Std 66.
+ request_uri = query and "?".join([path, query]) or path
+ scheme = scheme.lower()
+ defrag_uri = scheme + "://" + authority + request_uri
+ return scheme, authority, request_uri, defrag_uri
+
+
+# Cache filename construction (original borrowed from Venus http://intertwingly.net/code/venus/)
+re_url_scheme = re.compile(r'^\w+://')
+re_slash = re.compile(r'[?/:|]+')
+
+def safename(filename):
+ """Return a filename suitable for the cache.
+
+ Strips dangerous and common characters to create a filename we
+ can use to store the cache in.
+ """
+
+ try:
+ if re_url_scheme.match(filename):
+ if isinstance(filename,str):
+ filename = filename.decode('utf-8')
+ filename = filename.encode('idna')
+ else:
+ filename = filename.encode('idna')
+ except UnicodeError:
+ pass
+ if isinstance(filename,unicode):
+ filename=filename.encode('utf-8')
+ filemd5 = _md5(filename).hexdigest()
+ filename = re_url_scheme.sub("", filename)
+ filename = re_slash.sub(",", filename)
+
+ # limit length of filename
+ if len(filename)>200:
+ filename=filename[:200]
+ return ",".join((filename, filemd5))
+
+NORMALIZE_SPACE = re.compile(r'(?:\r\n)?[ \t]+')
+def _normalize_headers(headers):
+ return dict([ (key.lower(), NORMALIZE_SPACE.sub(value, ' ').strip()) for (key, value) in headers.iteritems()])
+
+def _parse_cache_control(headers):
+ retval = {}
+ if headers.has_key('cache-control'):
+ parts = headers['cache-control'].split(',')
+ parts_with_args = [tuple([x.strip().lower() for x in part.split("=", 1)]) for part in parts if -1 != part.find("=")]
+ parts_wo_args = [(name.strip().lower(), 1) for name in parts if -1 == name.find("=")]
+ retval = dict(parts_with_args + parts_wo_args)
+ return retval
+
+# Whether to use a strict mode to parse WWW-Authenticate headers
+# Might lead to bad results in case of ill-formed header value,
+# so disabled by default, falling back to relaxed parsing.
+# Set to true to turn on, usefull for testing servers.
+USE_WWW_AUTH_STRICT_PARSING = 0
+
+# In regex below:
+# [^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+ matches a "token" as defined by HTTP
+# "(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?" matches a "quoted-string" as defined by HTTP, when LWS have already been replaced by a single space
+# Actually, as an auth-param value can be either a token or a quoted-string, they are combined in a single pattern which matches both:
+# \"?((?<=\")(?:[^\0-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?<!\")[^\0-\x08\x0A-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+(?!\"))\"?
+WWW_AUTH_STRICT = re.compile(r"^(?:\s*(?:,\s*)?([^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+)\s*=\s*\"?((?<=\")(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?<!\")[^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+(?!\"))\"?)(.*)$")
+WWW_AUTH_RELAXED = re.compile(r"^(?:\s*(?:,\s*)?([^ \t\r\n=]+)\s*=\s*\"?((?<=\")(?:[^\\\"]|\\.)*?(?=\")|(?<!\")[^ \t\r\n,]+(?!\"))\"?)(.*)$")
+UNQUOTE_PAIRS = re.compile(r'\\(.)')
+def _parse_www_authenticate(headers, headername='www-authenticate'):
+ """Returns a dictionary of dictionaries, one dict
+ per auth_scheme."""
+ retval = {}
+ if headers.has_key(headername):
+ try:
+ authenticate = headers[headername].strip()
+ www_auth = USE_WWW_AUTH_STRICT_PARSING and WWW_AUTH_STRICT or WWW_AUTH_RELAXED
+ while authenticate:
+ # Break off the scheme at the beginning of the line
+ if headername == 'authentication-info':
+ (auth_scheme, the_rest) = ('digest', authenticate)
+ else:
+ (auth_scheme, the_rest) = authenticate.split(" ", 1)
+ # Now loop over all the key value pairs that come after the scheme,
+ # being careful not to roll into the next scheme
+ match = www_auth.search(the_rest)
+ auth_params = {}
+ while match:
+ if match and len(match.groups()) == 3:
+ (key, value, the_rest) = match.groups()
+ auth_params[key.lower()] = UNQUOTE_PAIRS.sub(r'\1', value) # '\\'.join([x.replace('\\', '') for x in value.split('\\\\')])
+ match = www_auth.search(the_rest)
+ retval[auth_scheme.lower()] = auth_params
+ authenticate = the_rest.strip()
+ except ValueError:
+ raise MalformedHeader("WWW-Authenticate")
+ return retval
+
+
+def _entry_disposition(response_headers, request_headers):
+ """Determine freshness from the Date, Expires and Cache-Control headers.
+
+ We don't handle the following:
+
+ 1. Cache-Control: max-stale
+ 2. Age: headers are not used in the calculations.
+
+ Not that this algorithm is simpler than you might think
+ because we are operating as a private (non-shared) cache.
+ This lets us ignore 's-maxage'. We can also ignore
+ 'proxy-invalidate' since we aren't a proxy.
+ We will never return a stale document as
+ fresh as a design decision, and thus the non-implementation
+ of 'max-stale'. This also lets us safely ignore 'must-revalidate'
+ since we operate as if every server has sent 'must-revalidate'.
+ Since we are private we get to ignore both 'public' and
+ 'private' parameters. We also ignore 'no-transform' since
+ we don't do any transformations.
+ The 'no-store' parameter is handled at a higher level.
+ So the only Cache-Control parameters we look at are:
+
+ no-cache
+ only-if-cached
+ max-age
+ min-fresh
+ """
+
+ retval = "STALE"
+ cc = _parse_cache_control(request_headers)
+ cc_response = _parse_cache_control(response_headers)
+
+ if request_headers.has_key('pragma') and request_headers['pragma'].lower().find('no-cache') != -1:
+ retval = "TRANSPARENT"
+ if 'cache-control' not in request_headers:
+ request_headers['cache-control'] = 'no-cache'
+ elif cc.has_key('no-cache'):
+ retval = "TRANSPARENT"
+ elif cc_response.has_key('no-cache'):
+ retval = "STALE"
+ elif cc.has_key('only-if-cached'):
+ retval = "FRESH"
+ elif response_headers.has_key('date'):
+ date = calendar.timegm(email.Utils.parsedate_tz(response_headers['date']))
+ now = time.time()
+ current_age = max(0, now - date)
+ if cc_response.has_key('max-age'):
+ try:
+ freshness_lifetime = int(cc_response['max-age'])
+ except ValueError:
+ freshness_lifetime = 0
+ elif response_headers.has_key('expires'):
+ expires = email.Utils.parsedate_tz(response_headers['expires'])
+ if None == expires:
+ freshness_lifetime = 0
+ else:
+ freshness_lifetime = max(0, calendar.timegm(expires) - date)
+ else:
+ freshness_lifetime = 0
+ if cc.has_key('max-age'):
+ try:
+ freshness_lifetime = int(cc['max-age'])
+ except ValueError:
+ freshness_lifetime = 0
+ if cc.has_key('min-fresh'):
+ try:
+ min_fresh = int(cc['min-fresh'])
+ except ValueError:
+ min_fresh = 0
+ current_age += min_fresh
+ if freshness_lifetime > current_age:
+ retval = "FRESH"
+ return retval
+
+def _decompressContent(response, new_content):
+ content = new_content
+ try:
+ encoding = response.get('content-encoding', None)
+ if encoding in ['gzip', 'deflate']:
+ if encoding == 'gzip':
+ content = gzip.GzipFile(fileobj=StringIO.StringIO(new_content)).read()
+ if encoding == 'deflate':
+ content = zlib.decompress(content)
+ response['content-length'] = str(len(content))
+ # Record the historical presence of the encoding in a way the won't interfere.
+ response['-content-encoding'] = response['content-encoding']
+ del response['content-encoding']
+ except IOError:
+ content = ""
+ raise FailedToDecompressContent(_("Content purported to be compressed with %s but failed to decompress.") % response.get('content-encoding'), response, content)
+ return content
+
+def _updateCache(request_headers, response_headers, content, cache, cachekey):
+ if cachekey:
+ cc = _parse_cache_control(request_headers)
+ cc_response = _parse_cache_control(response_headers)
+ if cc.has_key('no-store') or cc_response.has_key('no-store'):
+ cache.delete(cachekey)
+ else:
+ info = email.Message.Message()
+ for key, value in response_headers.iteritems():
+ if key not in ['status','content-encoding','transfer-encoding']:
+ info[key] = value
+
+ # Add annotations to the cache to indicate what headers
+ # are variant for this request.
+ vary = response_headers.get('vary', None)
+ if vary:
+ vary_headers = vary.lower().replace(' ', '').split(',')
+ for header in vary_headers:
+ key = '-varied-%s' % header
+ try:
+ info[key] = request_headers[header]
+ except KeyError:
+ pass
+
+ status = response_headers.status
+ if status == 304:
+ status = 200
+
+ status_header = 'status: %d\r\n' % status
+
+ header_str = info.as_string()
+
+ header_str = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", header_str)
+ text = "".join([status_header, header_str, content])
+
+ cache.set(cachekey, text)
+
+def _cnonce():
+ dig = _md5("%s:%s" % (time.ctime(), ["0123456789"[random.randrange(0, 9)] for i in range(20)])).hexdigest()
+ return dig[:16]
+
+def _wsse_username_token(cnonce, iso_now, password):
+ return base64.b64encode(_sha("%s%s%s" % (cnonce, iso_now, password)).digest()).strip()
+
+
+# For credentials we need two things, first
+# a pool of credential to try (not necesarily tied to BAsic, Digest, etc.)
+# Then we also need a list of URIs that have already demanded authentication
+# That list is tricky since sub-URIs can take the same auth, or the
+# auth scheme may change as you descend the tree.
+# So we also need each Auth instance to be able to tell us
+# how close to the 'top' it is.
+
+class Authentication(object):
+ def __init__(self, credentials, host, request_uri, headers, response, content, http):
+ (scheme, authority, path, query, fragment) = parse_uri(request_uri)
+ self.path = path
+ self.host = host
+ self.credentials = credentials
+ self.http = http
+
+ def depth(self, request_uri):
+ (scheme, authority, path, query, fragment) = parse_uri(request_uri)
+ return request_uri[len(self.path):].count("/")
+
+ def inscope(self, host, request_uri):
+ # XXX Should we normalize the request_uri?
+ (scheme, authority, path, query, fragment) = parse_uri(request_uri)
+ return (host == self.host) and path.startswith(self.path)
+
+ def request(self, method, request_uri, headers, content):
+ """Modify the request headers to add the appropriate
+ Authorization header. Over-ride this in sub-classes."""
+ pass
+
+ def response(self, response, content):
+ """Gives us a chance to update with new nonces
+ or such returned from the last authorized response.
+ Over-rise this in sub-classes if necessary.
+
+ Return TRUE is the request is to be retried, for
+ example Digest may return stale=true.
+ """
+ return False
+
+
+
+class BasicAuthentication(Authentication):
+ def __init__(self, credentials, host, request_uri, headers, response, content, http):
+ Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http)
+
+ def request(self, method, request_uri, headers, content):
+ """Modify the request headers to add the appropriate
+ Authorization header."""
+ headers['authorization'] = 'Basic ' + base64.b64encode("%s:%s" % self.credentials).strip()
+
+
+class DigestAuthentication(Authentication):
+ """Only do qop='auth' and MD5, since that
+ is all Apache currently implements"""
+ def __init__(self, credentials, host, request_uri, headers, response, content, http):
+ Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http)
+ challenge = _parse_www_authenticate(response, 'www-authenticate')
+ self.challenge = challenge['digest']
+ qop = self.challenge.get('qop', 'auth')
+ self.challenge['qop'] = ('auth' in [x.strip() for x in qop.split()]) and 'auth' or None
+ if self.challenge['qop'] is None:
+ raise UnimplementedDigestAuthOptionError( _("Unsupported value for qop: %s." % qop))
+ self.challenge['algorithm'] = self.challenge.get('algorithm', 'MD5').upper()
+ if self.challenge['algorithm'] != 'MD5':
+ raise UnimplementedDigestAuthOptionError( _("Unsupported value for algorithm: %s." % self.challenge['algorithm']))
+ self.A1 = "".join([self.credentials[0], ":", self.challenge['realm'], ":", self.credentials[1]])
+ self.challenge['nc'] = 1
+
+ def request(self, method, request_uri, headers, content, cnonce = None):
+ """Modify the request headers"""
+ H = lambda x: _md5(x).hexdigest()
+ KD = lambda s, d: H("%s:%s" % (s, d))
+ A2 = "".join([method, ":", request_uri])
+ self.challenge['cnonce'] = cnonce or _cnonce()
+ request_digest = '"%s"' % KD(H(self.A1), "%s:%s:%s:%s:%s" % (self.challenge['nonce'],
+ '%08x' % self.challenge['nc'],
+ self.challenge['cnonce'],
+ self.challenge['qop'], H(A2)
+ ))
+ headers['authorization'] = 'Digest username="%s", realm="%s", nonce="%s", uri="%s", algorithm=%s, response=%s, qop=%s, nc=%08x, cnonce="%s"' % (
+ self.credentials[0],
+ self.challenge['realm'],
+ self.challenge['nonce'],
+ request_uri,
+ self.challenge['algorithm'],
+ request_digest,
+ self.challenge['qop'],
+ self.challenge['nc'],
+ self.challenge['cnonce'],
+ )
+ if self.challenge.get('opaque'):
+ headers['authorization'] += ', opaque="%s"' % self.challenge['opaque']
+ self.challenge['nc'] += 1
+
+ def response(self, response, content):
+ if not response.has_key('authentication-info'):
+ challenge = _parse_www_authenticate(response, 'www-authenticate').get('digest', {})
+ if 'true' == challenge.get('stale'):
+ self.challenge['nonce'] = challenge['nonce']
+ self.challenge['nc'] = 1
+ return True
+ else:
+ updated_challenge = _parse_www_authenticate(response, 'authentication-info').get('digest', {})
+
+ if updated_challenge.has_key('nextnonce'):
+ self.challenge['nonce'] = updated_challenge['nextnonce']
+ self.challenge['nc'] = 1
+ return False
+
+
+class HmacDigestAuthentication(Authentication):
+ """Adapted from Robert Sayre's code and DigestAuthentication above."""
+ __author__ = "Thomas Broyer (t.broyer@ltgt.net)"
+
+ def __init__(self, credentials, host, request_uri, headers, response, content, http):
+ Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http)
+ challenge = _parse_www_authenticate(response, 'www-authenticate')
+ self.challenge = challenge['hmacdigest']
+ # TODO: self.challenge['domain']
+ self.challenge['reason'] = self.challenge.get('reason', 'unauthorized')
+ if self.challenge['reason'] not in ['unauthorized', 'integrity']:
+ self.challenge['reason'] = 'unauthorized'
+ self.challenge['salt'] = self.challenge.get('salt', '')
+ if not self.challenge.get('snonce'):
+ raise UnimplementedHmacDigestAuthOptionError( _("The challenge doesn't contain a server nonce, or this one is empty."))
+ self.challenge['algorithm'] = self.challenge.get('algorithm', 'HMAC-SHA-1')
+ if self.challenge['algorithm'] not in ['HMAC-SHA-1', 'HMAC-MD5']:
+ raise UnimplementedHmacDigestAuthOptionError( _("Unsupported value for algorithm: %s." % self.challenge['algorithm']))
+ self.challenge['pw-algorithm'] = self.challenge.get('pw-algorithm', 'SHA-1')
+ if self.challenge['pw-algorithm'] not in ['SHA-1', 'MD5']:
+ raise UnimplementedHmacDigestAuthOptionError( _("Unsupported value for pw-algorithm: %s." % self.challenge['pw-algorithm']))
+ if self.challenge['algorithm'] == 'HMAC-MD5':
+ self.hashmod = _md5
+ else:
+ self.hashmod = _sha
+ if self.challenge['pw-algorithm'] == 'MD5':
+ self.pwhashmod = _md5
+ else:
+ self.pwhashmod = _sha
+ self.key = "".join([self.credentials[0], ":",
+ self.pwhashmod.new("".join([self.credentials[1], self.challenge['salt']])).hexdigest().lower(),
+ ":", self.challenge['realm']
+ ])
+ self.key = self.pwhashmod.new(self.key).hexdigest().lower()
+
+ def request(self, method, request_uri, headers, content):
+ """Modify the request headers"""
+ keys = _get_end2end_headers(headers)
+ keylist = "".join(["%s " % k for k in keys])
+ headers_val = "".join([headers[k] for k in keys])
+ created = time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())
+ cnonce = _cnonce()
+ request_digest = "%s:%s:%s:%s:%s" % (method, request_uri, cnonce, self.challenge['snonce'], headers_val)
+ request_digest = hmac.new(self.key, request_digest, self.hashmod).hexdigest().lower()
+ headers['authorization'] = 'HMACDigest username="%s", realm="%s", snonce="%s", cnonce="%s", uri="%s", created="%s", response="%s", headers="%s"' % (
+ self.credentials[0],
+ self.challenge['realm'],
+ self.challenge['snonce'],
+ cnonce,
+ request_uri,
+ created,
+ request_digest,
+ keylist,
+ )
+
+ def response(self, response, content):
+ challenge = _parse_www_authenticate(response, 'www-authenticate').get('hmacdigest', {})
+ if challenge.get('reason') in ['integrity', 'stale']:
+ return True
+ return False
+
+
+class WsseAuthentication(Authentication):
+ """This is thinly tested and should not be relied upon.
+ At this time there isn't any third party server to test against.
+ Blogger and TypePad implemented this algorithm at one point
+ but Blogger has since switched to Basic over HTTPS and
+ TypePad has implemented it wrong, by never issuing a 401
+ challenge but instead requiring your client to telepathically know that
+ their endpoint is expecting WSSE profile="UsernameToken"."""
+ def __init__(self, credentials, host, request_uri, headers, response, content, http):
+ Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http)
+
+ def request(self, method, request_uri, headers, content):
+ """Modify the request headers to add the appropriate
+ Authorization header."""
+ headers['authorization'] = 'WSSE profile="UsernameToken"'
+ iso_now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+ cnonce = _cnonce()
+ password_digest = _wsse_username_token(cnonce, iso_now, self.credentials[1])
+ headers['X-WSSE'] = 'UsernameToken Username="%s", PasswordDigest="%s", Nonce="%s", Created="%s"' % (
+ self.credentials[0],
+ password_digest,
+ cnonce,
+ iso_now)
+
+class GoogleLoginAuthentication(Authentication):
+ def __init__(self, credentials, host, request_uri, headers, response, content, http):
+ from urllib import urlencode
+ Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http)
+ challenge = _parse_www_authenticate(response, 'www-authenticate')
+ service = challenge['googlelogin'].get('service', 'xapi')
+ # Bloggger actually returns the service in the challenge
+ # For the rest we guess based on the URI
+ if service == 'xapi' and request_uri.find("calendar") > 0:
+ service = "cl"
+ # No point in guessing Base or Spreadsheet
+ #elif request_uri.find("spreadsheets") > 0:
+ # service = "wise"
+
+ auth = dict(Email=credentials[0], Passwd=credentials[1], service=service, source=headers['user-agent'])
+ resp, content = self.http.request("https://www.google.com/accounts/ClientLogin", method="POST", body=urlencode(auth), headers={'Content-Type': 'application/x-www-form-urlencoded'})
+ lines = content.split('\n')
+ d = dict([tuple(line.split("=", 1)) for line in lines if line])
+ if resp.status == 403:
+ self.Auth = ""
+ else:
+ self.Auth = d['Auth']
+
+ def request(self, method, request_uri, headers, content):
+ """Modify the request headers to add the appropriate
+ Authorization header."""
+ headers['authorization'] = 'GoogleLogin Auth=' + self.Auth
+
+
+AUTH_SCHEME_CLASSES = {
+ "basic": BasicAuthentication,
+ "wsse": WsseAuthentication,
+ "digest": DigestAuthentication,
+ "hmacdigest": HmacDigestAuthentication,
+ "googlelogin": GoogleLoginAuthentication
+}
+
+AUTH_SCHEME_ORDER = ["hmacdigest", "googlelogin", "digest", "wsse", "basic"]
+
+class FileCache(object):
+ """Uses a local directory as a store for cached files.
+ Not really safe to use if multiple threads or processes are going to
+ be running on the same cache.
+ """
+ def __init__(self, cache, safe=safename): # use safe=lambda x: md5.new(x).hexdigest() for the old behavior
+ self.cache = cache
+ self.safe = safe
+ if not os.path.exists(cache):
+ os.makedirs(self.cache)
+
+ def get(self, key):
+ retval = None
+ cacheFullPath = os.path.join(self.cache, self.safe(key))
+ try:
+ f = file(cacheFullPath, "rb")
+ retval = f.read()
+ f.close()
+ except IOError:
+ pass
+ return retval
+
+ def set(self, key, value):
+ cacheFullPath = os.path.join(self.cache, self.safe(key))
+ f = file(cacheFullPath, "wb")
+ f.write(value)
+ f.close()
+
+ def delete(self, key):
+ cacheFullPath = os.path.join(self.cache, self.safe(key))
+ if os.path.exists(cacheFullPath):
+ os.remove(cacheFullPath)
+
+class Credentials(object):
+ def __init__(self):
+ self.credentials = []
+
+ def add(self, name, password, domain=""):
+ self.credentials.append((domain.lower(), name, password))
+
+ def clear(self):
+ self.credentials = []
+
+ def iter(self, domain):
+ for (cdomain, name, password) in self.credentials:
+ if cdomain == "" or domain == cdomain:
+ yield (name, password)
+
+class KeyCerts(Credentials):
+ """Identical to Credentials except that
+ name/password are mapped to key/cert."""
+ pass
+
+class AllHosts(object):
+ pass
+
+class ProxyInfo(object):
+ """Collect information required to use a proxy."""
+ bypass_hosts = ()
+
+ def __init__(self, proxy_type, proxy_host, proxy_port,
+ proxy_rdns=None, proxy_user=None, proxy_pass=None):
+ """The parameter proxy_type must be set to one of socks.PROXY_TYPE_XXX
+ constants. For example:
+
+ p = ProxyInfo(proxy_type=socks.PROXY_TYPE_HTTP,
+ proxy_host='localhost', proxy_port=8000)
+ """
+ self.proxy_type = proxy_type
+ self.proxy_host = proxy_host
+ self.proxy_port = proxy_port
+ self.proxy_rdns = proxy_rdns
+ self.proxy_user = proxy_user
+ self.proxy_pass = proxy_pass
+
+ def astuple(self):
+ return (self.proxy_type, self.proxy_host, self.proxy_port,
+ self.proxy_rdns, self.proxy_user, self.proxy_pass)
+
+ def isgood(self):
+ return (self.proxy_host != None) and (self.proxy_port != None)
+
+ def applies_to(self, hostname):
+ return not self.bypass_host(hostname)
+
+ def bypass_host(self, hostname):
+ """Has this host been excluded from the proxy config"""
+ if self.bypass_hosts is AllHosts:
+ return True
+
+ bypass = False
+ for domain in self.bypass_hosts:
+ if hostname.endswith(domain):
+ bypass = True
+
+ return bypass
+
+
+def proxy_info_from_environment(method='http'):
+ """
+ Read proxy info from the environment variables.
+ """
+ if method not in ['http', 'https']:
+ return
+
+ env_var = method + '_proxy'
+ url = os.environ.get(env_var, os.environ.get(env_var.upper()))
+ if not url:
+ return
+ pi = proxy_info_from_url(url, method)
+
+ no_proxy = os.environ.get('no_proxy', os.environ.get('NO_PROXY', ''))
+ bypass_hosts = []
+ if no_proxy:
+ bypass_hosts = no_proxy.split(',')
+ # special case, no_proxy=* means all hosts bypassed
+ if no_proxy == '*':
+ bypass_hosts = AllHosts
+
+ pi.bypass_hosts = bypass_hosts
+ return pi
+
+def proxy_info_from_url(url, method='http'):
+ """
+ Construct a ProxyInfo from a URL (such as http_proxy env var)
+ """
+ url = urlparse.urlparse(url)
+ username = None
+ password = None
+ port = None
+ if '@' in url[1]:
+ ident, host_port = url[1].split('@', 1)
+ if ':' in ident:
+ username, password = ident.split(':', 1)
+ else:
+ password = ident
+ else:
+ host_port = url[1]
+ if ':' in host_port:
+ host, port = host_port.split(':', 1)
+ else:
+ host = host_port
+
+ if port:
+ port = int(port)
+ else:
+ port = dict(https=443, http=80)[method]
+
+ proxy_type = 3 # socks.PROXY_TYPE_HTTP
+ return ProxyInfo(
+ proxy_type = proxy_type,
+ proxy_host = host,
+ proxy_port = port,
+ proxy_user = username or None,
+ proxy_pass = password or None,
+ )
+
+
+class HTTPConnectionWithTimeout(httplib.HTTPConnection):
+ """
+ HTTPConnection subclass that supports timeouts
+
+ All timeouts are in seconds. If None is passed for timeout then
+ Python's default timeout for sockets will be used. See for example
+ the docs of socket.setdefaulttimeout():
+ http://docs.python.org/library/socket.html#socket.setdefaulttimeout
+ """
+
+ def __init__(self, host, port=None, strict=None, timeout=None, proxy_info=None):
+ httplib.HTTPConnection.__init__(self, host, port, strict)
+ self.timeout = timeout
+ self.proxy_info = proxy_info
+
+ def connect(self):
+ """Connect to the host and port specified in __init__."""
+ # Mostly verbatim from httplib.py.
+ if self.proxy_info and socks is None:
+ raise ProxiesUnavailableError(
+ 'Proxy support missing but proxy use was requested!')
+ msg = "getaddrinfo returns an empty list"
+ if self.proxy_info and self.proxy_info.isgood():
+ use_proxy = True
+ proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass = self.proxy_info.astuple()
+ else:
+ use_proxy = False
+ if use_proxy and proxy_rdns:
+ host = proxy_host
+ port = proxy_port
+ else:
+ host = self.host
+ port = self.port
+
+ for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):
+ af, socktype, proto, canonname, sa = res
+ try:
+ if use_proxy:
+ self.sock = socks.socksocket(af, socktype, proto)
+ self.sock.setproxy(proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass)
+ else:
+ self.sock = socket.socket(af, socktype, proto)
+ self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+ # Different from httplib: support timeouts.
+ if has_timeout(self.timeout):
+ self.sock.settimeout(self.timeout)
+ # End of difference from httplib.
+ if self.debuglevel > 0:
+ print "connect: (%s, %s) ************" % (self.host, self.port)
+ if use_proxy:
+ print "proxy: %s ************" % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass))
+
+ self.sock.connect((self.host, self.port) + sa[2:])
+ except socket.error, msg:
+ if self.debuglevel > 0:
+ print "connect fail: (%s, %s)" % (self.host, self.port)
+ if use_proxy:
+ print "proxy: %s" % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass))
+ if self.sock:
+ self.sock.close()
+ self.sock = None
+ continue
+ break
+ if not self.sock:
+ raise socket.error, msg
+
+class HTTPSConnectionWithTimeout(httplib.HTTPSConnection):
+ """
+ This class allows communication via SSL.
+
+ All timeouts are in seconds. If None is passed for timeout then
+ Python's default timeout for sockets will be used. See for example
+ the docs of socket.setdefaulttimeout():
+ http://docs.python.org/library/socket.html#socket.setdefaulttimeout
+ """
+ def __init__(self, host, port=None, key_file=None, cert_file=None,
+ strict=None, timeout=None, proxy_info=None,
+ ca_certs=None, disable_ssl_certificate_validation=False):
+ httplib.HTTPSConnection.__init__(self, host, port=port, key_file=key_file,
+ cert_file=cert_file, strict=strict)
+ self.timeout = timeout
+ self.proxy_info = proxy_info
+ if ca_certs is None:
+ ca_certs = CA_CERTS
+ self.ca_certs = ca_certs
+ self.disable_ssl_certificate_validation = \
+ disable_ssl_certificate_validation
+
+ # The following two methods were adapted from https_wrapper.py, released
+ # with the Google Appengine SDK at
+ # http://googleappengine.googlecode.com/svn-history/r136/trunk/python/google/appengine/tools/https_wrapper.py
+ # under the following license:
+ #
+ # Copyright 2007 Google Inc.
+ #
+ # Licensed under the Apache License, Version 2.0 (the "License");
+ # you may not use this file except in compliance with the License.
+ # You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #
+
+ def _GetValidHostsForCert(self, cert):
+ """Returns a list of valid host globs for an SSL certificate.
+
+ Args:
+ cert: A dictionary representing an SSL certificate.
+ Returns:
+ list: A list of valid host globs.
+ """
+ if 'subjectAltName' in cert:
+ # Patch from richardfearn@gmail.com
+ # RFC 2459 states that subjectAltName may contain:
+ # either DNS, email, IP or URI
+ # email, URI,
+ hosts=[x[1] for x in cert['subjectAltName']
+ if x[0].lower() == "dns"]
+ if hosts:
+ return hosts
+
+ return [x[0][1] for x in cert['subject']
+ if x[0][0].lower() == 'commonname']
+
+ def _ValidateCertificateHostname(self, cert, hostname):
+ """Validates that a given hostname is valid for an SSL certificate.
+
+ Args:
+ cert: A dictionary representing an SSL certificate.
+ hostname: The hostname to test.
+ Returns:
+ bool: Whether or not the hostname is valid for this certificate.
+ """
+ hosts = self._GetValidHostsForCert(cert)
+ for host in hosts:
+ host_re = host.replace('.', '\.').replace('*', '[^.]*')
+ if re.search('^%s$' % (host_re,), hostname, re.I):
+ return True
+ return False
+
+ def connect(self):
+ "Connect to a host on a given (SSL) port."
+
+ msg = "getaddrinfo returns an empty list"
+ if self.proxy_info and self.proxy_info.isgood():
+ use_proxy = True
+ proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass = self.proxy_info.astuple()
+ else:
+ use_proxy = False
+ if use_proxy and proxy_rdns:
+ host = proxy_host
+ port = proxy_port
+ else:
+ host = self.host
+ port = self.port
+
+ for family, socktype, proto, canonname, sockaddr in socket.getaddrinfo(
+ host, port, 0, socket.SOCK_STREAM):
+ try:
+ if use_proxy:
+ sock = socks.socksocket(family, socktype, proto)
+
+ sock.setproxy(proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass)
+ else:
+ sock = socket.socket(family, socktype, proto)
+ sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+
+ if has_timeout(self.timeout):
+ sock.settimeout(self.timeout)
+ sock.connect((self.host, self.port))
+ self.sock =_ssl_wrap_socket(
+ sock, self.key_file, self.cert_file,
+ self.disable_ssl_certificate_validation, self.ca_certs)
+ if self.debuglevel > 0:
+ print "connect: (%s, %s)" % (self.host, self.port)
+ if use_proxy:
+ print "proxy: %s" % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass))
+ if not self.disable_ssl_certificate_validation:
+ cert = self.sock.getpeercert()
+ hostname = self.host.split(':', 0)[0]
+ if not self._ValidateCertificateHostname(cert, hostname):
+ raise CertificateHostnameMismatch(
+ 'Server presented certificate that does not match '
+ 'host %s: %s' % (hostname, cert), hostname, cert)
+ except ssl_SSLError, e:
+ if sock:
+ sock.close()
+ if self.sock:
+ self.sock.close()
+ self.sock = None
+ # Unfortunately the ssl module doesn't seem to provide any way
+ # to get at more detailed error information, in particular
+ # whether the error is due to certificate validation or
+ # something else (such as SSL protocol mismatch).
+ if e.errno == ssl.SSL_ERROR_SSL:
+ raise SSLHandshakeError(e)
+ else:
+ raise
+ except (socket.timeout, socket.gaierror):
+ raise
+ except socket.error, msg:
+ if self.debuglevel > 0:
+ print "connect fail: (%s, %s)" % (self.host, self.port)
+ if use_proxy:
+ print "proxy: %s" % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass))
+ if self.sock:
+ self.sock.close()
+ self.sock = None
+ continue
+ break
+ if not self.sock:
+ raise socket.error, msg
+
+SCHEME_TO_CONNECTION = {
+ 'http': HTTPConnectionWithTimeout,
+ 'https': HTTPSConnectionWithTimeout
+ }
+
+# Use a different connection object for Google App Engine
+try:
+ from google.appengine.api import apiproxy_stub_map
+ if apiproxy_stub_map.apiproxy.GetStub('urlfetch') is None:
+ raise ImportError # Bail out; we're not actually running on App Engine.
+ from google.appengine.api.urlfetch import fetch
+ from google.appengine.api.urlfetch import InvalidURLError
+
+ class ResponseDict(dict):
+ """Dictionary with a read() method; can pass off as httplib.HTTPResponse."""
+ def __init__(self, *args, **kwargs):
+ self.content = kwargs.pop('content', None)
+ return super(ResponseDict, self).__init__(*args, **kwargs)
+
+ def read(self):
+ return self.content
+
+
+ class AppEngineHttpConnection(object):
+ """Emulates an httplib.HTTPConnection object, but actually uses the Google
+ App Engine urlfetch library. This allows the timeout to be properly used on
+ Google App Engine, and avoids using httplib, which on Google App Engine is
+ just another wrapper around urlfetch.
+ """
+ def __init__(self, host, port=None, key_file=None, cert_file=None,
+ strict=None, timeout=None, proxy_info=None, ca_certs=None,
+ disable_ssl_certificate_validation=False):
+ self.host = host
+ self.port = port
+ self.timeout = timeout
+ if key_file or cert_file or proxy_info or ca_certs:
+ raise NotSupportedOnThisPlatform()
+ self.response = None
+ self.scheme = 'http'
+ self.validate_certificate = not disable_ssl_certificate_validation
+ self.sock = True
+
+ def request(self, method, url, body, headers):
+ # Calculate the absolute URI, which fetch requires
+ netloc = self.host
+ if self.port:
+ netloc = '%s:%s' % (self.host, self.port)
+ absolute_uri = '%s://%s%s' % (self.scheme, netloc, url)
+ try:
+ try: # 'body' can be a stream.
+ body = body.read()
+ except AttributeError:
+ pass
+ response = fetch(absolute_uri, payload=body, method=method,
+ headers=headers, allow_truncated=False, follow_redirects=False,
+ deadline=self.timeout,
+ validate_certificate=self.validate_certificate)
+ self.response = ResponseDict(response.headers, content=response.content)
+ self.response['status'] = str(response.status_code)
+ self.response['reason'] = httplib.responses.get(response.status_code, 'Ok')
+ self.response.status = response.status_code
+
+ # Make sure the exceptions raised match the exceptions expected.
+ except InvalidURLError:
+ raise socket.gaierror('')
+
+ def getresponse(self):
+ if self.response:
+ return self.response
+ else:
+ raise httplib.HTTPException()
+
+ def set_debuglevel(self, level):
+ pass
+
+ def connect(self):
+ pass
+
+ def close(self):
+ pass
+
+
+ class AppEngineHttpsConnection(AppEngineHttpConnection):
+ """Same as AppEngineHttpConnection, but for HTTPS URIs."""
+ def __init__(self, host, port=None, key_file=None, cert_file=None,
+ strict=None, timeout=None, proxy_info=None, ca_certs=None,
+ disable_ssl_certificate_validation=False):
+ AppEngineHttpConnection.__init__(self, host, port, key_file, cert_file,
+ strict, timeout, proxy_info, ca_certs, disable_ssl_certificate_validation)
+ self.scheme = 'https'
+
+ # Update the connection classes to use the Googel App Engine specific ones.
+ SCHEME_TO_CONNECTION = {
+ 'http': AppEngineHttpConnection,
+ 'https': AppEngineHttpsConnection
+ }
+
+except ImportError:
+ pass
+
+
+class Http(object):
+ """An HTTP client that handles:
+- all methods
+- caching
+- ETags
+- compression,
+- HTTPS
+- Basic
+- Digest
+- WSSE
+
+and more.
+ """
+ def __init__(self, cache=None, timeout=None,
+ proxy_info=proxy_info_from_environment,
+ ca_certs=None, disable_ssl_certificate_validation=False):
+ """If 'cache' is a string then it is used as a directory name for
+ a disk cache. Otherwise it must be an object that supports the
+ same interface as FileCache.
+
+ All timeouts are in seconds. If None is passed for timeout
+ then Python's default timeout for sockets will be used. See
+ for example the docs of socket.setdefaulttimeout():
+ http://docs.python.org/library/socket.html#socket.setdefaulttimeout
+
+ `proxy_info` may be:
+ - a callable that takes the http scheme ('http' or 'https') and
+ returns a ProxyInfo instance per request. By default, uses
+ proxy_nfo_from_environment.
+ - a ProxyInfo instance (static proxy config).
+ - None (proxy disabled).
+
+ ca_certs is the path of a file containing root CA certificates for SSL
+ server certificate validation. By default, a CA cert file bundled with
+ httplib2 is used.
+
+ If disable_ssl_certificate_validation is true, SSL cert validation will
+ not be performed.
+ """
+ self.proxy_info = proxy_info
+ self.ca_certs = ca_certs
+ self.disable_ssl_certificate_validation = \
+ disable_ssl_certificate_validation
+
+ # Map domain name to an httplib connection
+ self.connections = {}
+ # The location of the cache, for now a directory
+ # where cached responses are held.
+ if cache and isinstance(cache, basestring):
+ self.cache = FileCache(cache)
+ else:
+ self.cache = cache
+
+ # Name/password
+ self.credentials = Credentials()
+
+ # Key/cert
+ self.certificates = KeyCerts()
+
+ # authorization objects
+ self.authorizations = []
+
+ # If set to False then no redirects are followed, even safe ones.
+ self.follow_redirects = True
+
+ # Which HTTP methods do we apply optimistic concurrency to, i.e.
+ # which methods get an "if-match:" etag header added to them.
+ self.optimistic_concurrency_methods = ["PUT", "PATCH"]
+
+ # If 'follow_redirects' is True, and this is set to True then
+ # all redirecs are followed, including unsafe ones.
+ self.follow_all_redirects = False
+
+ self.ignore_etag = False
+
+ self.force_exception_to_status_code = False
+
+ self.timeout = timeout
+
+ # Keep Authorization: headers on a redirect.
+ self.forward_authorization_headers = False
+
+ def __getstate__(self):
+ state_dict = copy.copy(self.__dict__)
+ # In case request is augmented by some foreign object such as
+ # credentials which handle auth
+ if 'request' in state_dict:
+ del state_dict['request']
+ if 'connections' in state_dict:
+ del state_dict['connections']
+ return state_dict
+
+ def __setstate__(self, state):
+ self.__dict__.update(state)
+ self.connections = {}
+
+ def _auth_from_challenge(self, host, request_uri, headers, response, content):
+ """A generator that creates Authorization objects
+ that can be applied to requests.
+ """
+ challenges = _parse_www_authenticate(response, 'www-authenticate')
+ for cred in self.credentials.iter(host):
+ for scheme in AUTH_SCHEME_ORDER:
+ if challenges.has_key(scheme):
+ yield AUTH_SCHEME_CLASSES[scheme](cred, host, request_uri, headers, response, content, self)
+
+ def add_credentials(self, name, password, domain=""):
+ """Add a name and password that will be used
+ any time a request requires authentication."""
+ self.credentials.add(name, password, domain)
+
+ def add_certificate(self, key, cert, domain):
+ """Add a key and cert that will be used
+ any time a request requires authentication."""
+ self.certificates.add(key, cert, domain)
+
+ def clear_credentials(self):
+ """Remove all the names and passwords
+ that are used for authentication"""
+ self.credentials.clear()
+ self.authorizations = []
+
+ def _conn_request(self, conn, request_uri, method, body, headers):
+ for i in range(RETRIES):
+ try:
+ if conn.sock is None:
+ conn.connect()
+ conn.request(method, request_uri, body, headers)
+ except socket.timeout:
+ raise
+ except socket.gaierror:
+ conn.close()
+ raise ServerNotFoundError("Unable to find the server at %s" % conn.host)
+ except ssl_SSLError:
+ conn.close()
+ raise
+ except socket.error, e:
+ err = 0
+ if hasattr(e, 'args'):
+ err = getattr(e, 'args')[0]
+ else:
+ err = e.errno
+ if err == errno.ECONNREFUSED: # Connection refused
+ raise
+ except httplib.HTTPException:
+ # Just because the server closed the connection doesn't apparently mean
+ # that the server didn't send a response.
+ if conn.sock is None:
+ if i < RETRIES-1:
+ conn.close()
+ conn.connect()
+ continue
+ else:
+ conn.close()
+ raise
+ if i < RETRIES-1:
+ conn.close()
+ conn.connect()
+ continue
+ try:
+ response = conn.getresponse()
+ except (socket.error, httplib.HTTPException):
+ if i < RETRIES-1:
+ conn.close()
+ conn.connect()
+ continue
+ else:
+ raise
+ else:
+ content = ""
+ if method == "HEAD":
+ conn.close()
+ else:
+ content = response.read()
+ response = Response(response)
+ if method != "HEAD":
+ content = _decompressContent(response, content)
+ break
+ return (response, content)
+
+
+ def _request(self, conn, host, absolute_uri, request_uri, method, body, headers, redirections, cachekey):
+ """Do the actual request using the connection object
+ and also follow one level of redirects if necessary"""
+
+ auths = [(auth.depth(request_uri), auth) for auth in self.authorizations if auth.inscope(host, request_uri)]
+ auth = auths and sorted(auths)[0][1] or None
+ if auth:
+ auth.request(method, request_uri, headers, body)
+
+ (response, content) = self._conn_request(conn, request_uri, method, body, headers)
+
+ if auth:
+ if auth.response(response, body):
+ auth.request(method, request_uri, headers, body)
+ (response, content) = self._conn_request(conn, request_uri, method, body, headers )
+ response._stale_digest = 1
+
+ if response.status == 401:
+ for authorization in self._auth_from_challenge(host, request_uri, headers, response, content):
+ authorization.request(method, request_uri, headers, body)
+ (response, content) = self._conn_request(conn, request_uri, method, body, headers, )
+ if response.status != 401:
+ self.authorizations.append(authorization)
+ authorization.response(response, body)
+ break
+
+ if (self.follow_all_redirects or (method in ["GET", "HEAD"]) or response.status == 303):
+ if self.follow_redirects and response.status in [300, 301, 302, 303, 307]:
+ # Pick out the location header and basically start from the beginning
+ # remembering first to strip the ETag header and decrement our 'depth'
+ if redirections:
+ if not response.has_key('location') and response.status != 300:
+ raise RedirectMissingLocation( _("Redirected but the response is missing a Location: header."), response, content)
+ # Fix-up relative redirects (which violate an RFC 2616 MUST)
+ if response.has_key('location'):
+ location = response['location']
+ (scheme, authority, path, query, fragment) = parse_uri(location)
+ if authority == None:
+ response['location'] = urlparse.urljoin(absolute_uri, location)
+ if response.status == 301 and method in ["GET", "HEAD"]:
+ response['-x-permanent-redirect-url'] = response['location']
+ if not response.has_key('content-location'):
+ response['content-location'] = absolute_uri
+ _updateCache(headers, response, content, self.cache, cachekey)
+ if headers.has_key('if-none-match'):
+ del headers['if-none-match']
+ if headers.has_key('if-modified-since'):
+ del headers['if-modified-since']
+ if 'authorization' in headers and not self.forward_authorization_headers:
+ del headers['authorization']
+ if response.has_key('location'):
+ location = response['location']
+ old_response = copy.deepcopy(response)
+ if not old_response.has_key('content-location'):
+ old_response['content-location'] = absolute_uri
+ redirect_method = method
+ if response.status in [302, 303]:
+ redirect_method = "GET"
+ body = None
+ (response, content) = self.request(location, redirect_method, body=body, headers = headers, redirections = redirections - 1)
+ response.previous = old_response
+ else:
+ raise RedirectLimit("Redirected more times than rediection_limit allows.", response, content)
+ elif response.status in [200, 203] and method in ["GET", "HEAD"]:
+ # Don't cache 206's since we aren't going to handle byte range requests
+ if not response.has_key('content-location'):
+ response['content-location'] = absolute_uri
+ _updateCache(headers, response, content, self.cache, cachekey)
+
+ return (response, content)
+
+ def _normalize_headers(self, headers):
+ return _normalize_headers(headers)
+
+# Need to catch and rebrand some exceptions
+# Then need to optionally turn all exceptions into status codes
+# including all socket.* and httplib.* exceptions.
+
+
+ def request(self, uri, method="GET", body=None, headers=None, redirections=DEFAULT_MAX_REDIRECTS, connection_type=None):
+ """ Performs a single HTTP request.
+The 'uri' is the URI of the HTTP resource and can begin
+with either 'http' or 'https'. The value of 'uri' must be an absolute URI.
+
+The 'method' is the HTTP method to perform, such as GET, POST, DELETE, etc.
+There is no restriction on the methods allowed.
+
+The 'body' is the entity body to be sent with the request. It is a string
+object.
+
+Any extra headers that are to be sent with the request should be provided in the
+'headers' dictionary.
+
+The maximum number of redirect to follow before raising an
+exception is 'redirections. The default is 5.
+
+The return value is a tuple of (response, content), the first
+being and instance of the 'Response' class, the second being
+a string that contains the response entity body.
+ """
+ try:
+ if headers is None:
+ headers = {}
+ else:
+ headers = self._normalize_headers(headers)
+
+ if not headers.has_key('user-agent'):
+ headers['user-agent'] = "Python-httplib2/%s (gzip)" % __version__
+
+ uri = iri2uri(uri)
+
+ (scheme, authority, request_uri, defrag_uri) = urlnorm(uri)
+ domain_port = authority.split(":")[0:2]
+ if len(domain_port) == 2 and domain_port[1] == '443' and scheme == 'http':
+ scheme = 'https'
+ authority = domain_port[0]
+
+ proxy_info = self._get_proxy_info(scheme, authority)
+
+ conn_key = scheme+":"+authority
+ if conn_key in self.connections:
+ conn = self.connections[conn_key]
+ else:
+ if not connection_type:
+ connection_type = SCHEME_TO_CONNECTION[scheme]
+ certs = list(self.certificates.iter(authority))
+ if scheme == 'https':
+ if certs:
+ conn = self.connections[conn_key] = connection_type(
+ authority, key_file=certs[0][0],
+ cert_file=certs[0][1], timeout=self.timeout,
+ proxy_info=proxy_info,
+ ca_certs=self.ca_certs,
+ disable_ssl_certificate_validation=
+ self.disable_ssl_certificate_validation)
+ else:
+ conn = self.connections[conn_key] = connection_type(
+ authority, timeout=self.timeout,
+ proxy_info=proxy_info,
+ ca_certs=self.ca_certs,
+ disable_ssl_certificate_validation=
+ self.disable_ssl_certificate_validation)
+ else:
+ conn = self.connections[conn_key] = connection_type(
+ authority, timeout=self.timeout,
+ proxy_info=proxy_info)
+ conn.set_debuglevel(debuglevel)
+
+ if 'range' not in headers and 'accept-encoding' not in headers:
+ headers['accept-encoding'] = 'gzip, deflate'
+
+ info = email.Message.Message()
+ cached_value = None
+ if self.cache:
+ cachekey = defrag_uri
+ cached_value = self.cache.get(cachekey)
+ if cached_value:
+ # info = email.message_from_string(cached_value)
+ #
+ # Need to replace the line above with the kludge below
+ # to fix the non-existent bug not fixed in this
+ # bug report: http://mail.python.org/pipermail/python-bugs-list/2005-September/030289.html
+ try:
+ info, content = cached_value.split('\r\n\r\n', 1)
+ feedparser = email.FeedParser.FeedParser()
+ feedparser.feed(info)
+ info = feedparser.close()
+ feedparser._parse = None
+ except (IndexError, ValueError):
+ self.cache.delete(cachekey)
+ cachekey = None
+ cached_value = None
+ else:
+ cachekey = None
+
+ if method in self.optimistic_concurrency_methods and self.cache and info.has_key('etag') and not self.ignore_etag and 'if-match' not in headers:
+ # http://www.w3.org/1999/04/Editing/
+ headers['if-match'] = info['etag']
+
+ if method not in ["GET", "HEAD"] and self.cache and cachekey:
+ # RFC 2616 Section 13.10
+ self.cache.delete(cachekey)
+
+ # Check the vary header in the cache to see if this request
+ # matches what varies in the cache.
+ if method in ['GET', 'HEAD'] and 'vary' in info:
+ vary = info['vary']
+ vary_headers = vary.lower().replace(' ', '').split(',')
+ for header in vary_headers:
+ key = '-varied-%s' % header
+ value = info[key]
+ if headers.get(header, None) != value:
+ cached_value = None
+ break
+
+ if cached_value and method in ["GET", "HEAD"] and self.cache and 'range' not in headers:
+ if info.has_key('-x-permanent-redirect-url'):
+ # Should cached permanent redirects be counted in our redirection count? For now, yes.
+ if redirections <= 0:
+ raise RedirectLimit("Redirected more times than rediection_limit allows.", {}, "")
+ (response, new_content) = self.request(info['-x-permanent-redirect-url'], "GET", headers = headers, redirections = redirections - 1)
+ response.previous = Response(info)
+ response.previous.fromcache = True
+ else:
+ # Determine our course of action:
+ # Is the cached entry fresh or stale?
+ # Has the client requested a non-cached response?
+ #
+ # There seems to be three possible answers:
+ # 1. [FRESH] Return the cache entry w/o doing a GET
+ # 2. [STALE] Do the GET (but add in cache validators if available)
+ # 3. [TRANSPARENT] Do a GET w/o any cache validators (Cache-Control: no-cache) on the request
+ entry_disposition = _entry_disposition(info, headers)
+
+ if entry_disposition == "FRESH":
+ if not cached_value:
+ info['status'] = '504'
+ content = ""
+ response = Response(info)
+ if cached_value:
+ response.fromcache = True
+ return (response, content)
+
+ if entry_disposition == "STALE":
+ if info.has_key('etag') and not self.ignore_etag and not 'if-none-match' in headers:
+ headers['if-none-match'] = info['etag']
+ if info.has_key('last-modified') and not 'last-modified' in headers:
+ headers['if-modified-since'] = info['last-modified']
+ elif entry_disposition == "TRANSPARENT":
+ pass
+
+ (response, new_content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
+
+ if response.status == 304 and method == "GET":
+ # Rewrite the cache entry with the new end-to-end headers
+ # Take all headers that are in response
+ # and overwrite their values in info.
+ # unless they are hop-by-hop, or are listed in the connection header.
+
+ for key in _get_end2end_headers(response):
+ info[key] = response[key]
+ merged_response = Response(info)
+ if hasattr(response, "_stale_digest"):
+ merged_response._stale_digest = response._stale_digest
+ _updateCache(headers, merged_response, content, self.cache, cachekey)
+ response = merged_response
+ response.status = 200
+ response.fromcache = True
+
+ elif response.status == 200:
+ content = new_content
+ else:
+ self.cache.delete(cachekey)
+ content = new_content
+ else:
+ cc = _parse_cache_control(headers)
+ if cc.has_key('only-if-cached'):
+ info['status'] = '504'
+ response = Response(info)
+ content = ""
+ else:
+ (response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
+ except Exception, e:
+ if self.force_exception_to_status_code:
+ if isinstance(e, HttpLib2ErrorWithResponse):
+ response = e.response
+ content = e.content
+ response.status = 500
+ response.reason = str(e)
+ elif isinstance(e, socket.timeout):
+ content = "Request Timeout"
+ response = Response( {
+ "content-type": "text/plain",
+ "status": "408",
+ "content-length": len(content)
+ })
+ response.reason = "Request Timeout"
+ else:
+ content = str(e)
+ response = Response( {
+ "content-type": "text/plain",
+ "status": "400",
+ "content-length": len(content)
+ })
+ response.reason = "Bad Request"
+ else:
+ raise
+
+
+ return (response, content)
+
+ def _get_proxy_info(self, scheme, authority):
+ """Return a ProxyInfo instance (or None) based on the scheme
+ and authority.
+ """
+ hostname, port = urllib.splitport(authority)
+ proxy_info = self.proxy_info
+ if callable(proxy_info):
+ proxy_info = proxy_info(scheme)
+
+ if (hasattr(proxy_info, 'applies_to')
+ and not proxy_info.applies_to(hostname)):
+ proxy_info = None
+ return proxy_info
+
+
+class Response(dict):
+ """An object more like email.Message than httplib.HTTPResponse."""
+
+ """Is this response from our local cache"""
+ fromcache = False
+
+ """HTTP protocol version used by server. 10 for HTTP/1.0, 11 for HTTP/1.1. """
+ version = 11
+
+ "Status code returned by server. "
+ status = 200
+
+ """Reason phrase returned by server."""
+ reason = "Ok"
+
+ previous = None
+
+ def __init__(self, info):
+ # info is either an email.Message or
+ # an httplib.HTTPResponse object.
+ if isinstance(info, httplib.HTTPResponse):
+ for key, value in info.getheaders():
+ self[key.lower()] = value
+ self.status = info.status
+ self['status'] = str(self.status)
+ self.reason = info.reason
+ self.version = info.version
+ elif isinstance(info, email.Message.Message):
+ for key, value in info.items():
+ self[key.lower()] = value
+ self.status = int(self['status'])
+ else:
+ for key, value in info.iteritems():
+ self[key.lower()] = value
+ self.status = int(self.get('status', self.status))
+ self.reason = self.get('reason', self.reason)
+
+
+ def __getattr__(self, name):
+ if name == 'dict':
+ return self
+ else:
+ raise AttributeError, name
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/702b6ddd/install/python-installer/scripts/httplib2/iri2uri.py
----------------------------------------------------------------------
diff --git a/install/python-installer/scripts/httplib2/iri2uri.py b/install/python-installer/scripts/httplib2/iri2uri.py
new file mode 100644
index 0000000..73b63f0
--- /dev/null
+++ b/install/python-installer/scripts/httplib2/iri2uri.py
@@ -0,0 +1,132 @@
+"""
+iri2uri
+
+Copyright (c) 2006 by Joe Gregorio
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without restriction,
+including without limitation the rights to use, copy, modify, merge,
+publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+Converts an IRI to a URI.
+
+"""
+__author__ = "Joe Gregorio (joe@bitworking.org)"
+__copyright__ = "Copyright 2006, Joe Gregorio"
+__contributors__ = []
+__version__ = "1.0.0"
+__license__ = "MIT"
+__history__ = """
+"""
+
+import urlparse
+
+
+# Convert an IRI to a URI following the rules in RFC 3987
+#
+# The characters we need to enocde and escape are defined in the spec:
+#
+# iprivate = %xE000-F8FF / %xF0000-FFFFD / %x100000-10FFFD
+# ucschar = %xA0-D7FF / %xF900-FDCF / %xFDF0-FFEF
+# / %x10000-1FFFD / %x20000-2FFFD / %x30000-3FFFD
+# / %x40000-4FFFD / %x50000-5FFFD / %x60000-6FFFD
+# / %x70000-7FFFD / %x80000-8FFFD / %x90000-9FFFD
+# / %xA0000-AFFFD / %xB0000-BFFFD / %xC0000-CFFFD
+# / %xD0000-DFFFD / %xE1000-EFFFD
+
+escape_range = [
+ (0xA0, 0xD7FF ),
+ (0xE000, 0xF8FF ),
+ (0xF900, 0xFDCF ),
+ (0xFDF0, 0xFFEF),
+ (0x10000, 0x1FFFD ),
+ (0x20000, 0x2FFFD ),
+ (0x30000, 0x3FFFD),
+ (0x40000, 0x4FFFD ),
+ (0x50000, 0x5FFFD ),
+ (0x60000, 0x6FFFD),
+ (0x70000, 0x7FFFD ),
+ (0x80000, 0x8FFFD ),
+ (0x90000, 0x9FFFD),
+ (0xA0000, 0xAFFFD ),
+ (0xB0000, 0xBFFFD ),
+ (0xC0000, 0xCFFFD),
+ (0xD0000, 0xDFFFD ),
+ (0xE1000, 0xEFFFD),
+ (0xF0000, 0xFFFFD ),
+ (0x100000, 0x10FFFD)
+]
+
+def encode(c):
+ retval = c
+ i = ord(c)
+ for low, high in escape_range:
+ if i < low:
+ break
+ if i >= low and i <= high:
+ retval = "".join(["%%%2X" % ord(o) for o in c.encode('utf-8')])
+ break
+ return retval
+
+
+def iri2uri(uri):
+ """Convert an IRI to a URI. Note that IRIs must be
+ passed in a unicode strings. That is, do not utf-8 encode
+ the IRI before passing it into the function."""
+ if isinstance(uri ,unicode):
+ (scheme, authority, path, query, fragment) = urlparse.urlsplit(uri)
+ authority = authority.encode('idna')
+ # For each character in 'ucschar' or 'iprivate'
+ # 1. encode as utf-8
+ # 2. then %-encode each octet of that utf-8
+ uri = urlparse.urlunsplit((scheme, authority, path, query, fragment))
+ uri = "".join([encode(c) for c in uri])
+ return uri
+
+if __name__ == "__main__":
+ import unittest
+
+ class Test(unittest.TestCase):
+
+ def test_uris(self):
+ """Test that URIs are invariant under the transformation."""
+ invariant = [
+ u"ftp://ftp.is.co.za/rfc/rfc1808.txt",
+ u"http://www.ietf.org/rfc/rfc2396.txt",
+ u"ldap://[2001:db8::7]/c=GB?objectClass?one",
+ u"mailto:John.Doe@example.com",
+ u"news:comp.infosystems.www.servers.unix",
+ u"tel:+1-816-555-1212",
+ u"telnet://192.0.2.16:80/",
+ u"urn:oasis:names:specification:docbook:dtd:xml:4.1.2" ]
+ for uri in invariant:
+ self.assertEqual(uri, iri2uri(uri))
+
+ def test_iri(self):
+ """ Test that the right type of escaping is done for each part of the URI."""
+ self.assertEqual("http://xn--o3h.com/%E2%98%84", iri2uri(u"http://\N{COMET}.com/\N{COMET}"))
+ self.assertEqual("http://bitworking.org/?fred=%E2%98%84", iri2uri(u"http://bitworking.org/?fred=\N{COMET}"))
+ self.assertEqual("http://bitworking.org/#%E2%98%84", iri2uri(u"http://bitworking.org/#\N{COMET}"))
+ self.assertEqual("#%E2%98%84", iri2uri(u"#\N{COMET}"))
+ self.assertEqual("/fred?bar=%E2%98%9A#%E2%98%84", iri2uri(u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}"))
+ self.assertEqual("/fred?bar=%E2%98%9A#%E2%98%84", iri2uri(iri2uri(u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}")))
+ self.assertNotEqual("/fred?bar=%E2%98%9A#%E2%98%84", iri2uri(u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}".encode('utf-8')))
+
+ unittest.main()
+
+
[3/5] incubator-trafodion git commit: [TRAFODION-2431] python
installer - add python lib dependencies
Posted by li...@apache.org.
[TRAFODION-2431] python installer - add python lib dependencies
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/702b6ddd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/702b6ddd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/702b6ddd
Branch: refs/heads/master
Commit: 702b6ddd25c11efff85f46e42ffbd6322119f514
Parents: 73446d8
Author: Eason <hf...@gmail.com>
Authored: Tue Jan 10 11:37:23 2017 +0800
Committer: Eason <hf...@gmail.com>
Committed: Wed Jan 11 21:20:23 2017 +0800
----------------------------------------------------------------------
install/Makefile | 5 +-
install/python-installer/prettytable.py | 1475 +++++++++++++++
.../scripts/httplib2/__init__.py | 1714 ++++++++++++++++++
.../scripts/httplib2/iri2uri.py | 132 ++
.../python-installer/scripts/httplib2/socks.py | 438 +++++
licenses/Makefile | 7 +-
licenses/lic-pyinstall-src | 63 +
7 files changed, 3831 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/702b6ddd/install/Makefile
----------------------------------------------------------------------
diff --git a/install/Makefile b/install/Makefile
index c6f5da6..5dc7048 100644
--- a/install/Makefile
+++ b/install/Makefile
@@ -47,7 +47,10 @@ installer/NOTICE: ../NOTICE
installer/DISCLAIMER: ../DISCLAIMER
cp -f $? $@
-python-installer/LICENSE: ../licenses/LICENSE-install
+../licenses/LICENSE-pyinstall:
+ cd $(@D) && $(MAKE) $(@F)
+
+python-installer/LICENSE: ../licenses/LICENSE-pyinstall
cp -f $? $@
python-installer/NOTICE: ../NOTICE
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/702b6ddd/install/python-installer/prettytable.py
----------------------------------------------------------------------
diff --git a/install/python-installer/prettytable.py b/install/python-installer/prettytable.py
new file mode 100644
index 0000000..8abb952
--- /dev/null
+++ b/install/python-installer/prettytable.py
@@ -0,0 +1,1475 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2009-2013, Luke Maurits <lu...@maurits.id.au>
+# All rights reserved.
+# With contributions from:
+# * Chris Clark
+# * Klein Stephane
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+# * The name of the author may not be used to endorse or promote products
+# derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+__version__ = "0.7.2"
+
+import copy
+import csv
+import random
+import re
+import sys
+import textwrap
+import itertools
+import unicodedata
+
+py3k = sys.version_info[0] >= 3
+if py3k:
+ unicode = str
+ basestring = str
+ itermap = map
+ iterzip = zip
+ uni_chr = chr
+ from html.parser import HTMLParser
+else:
+ itermap = itertools.imap
+ iterzip = itertools.izip
+ uni_chr = unichr
+ from HTMLParser import HTMLParser
+
+if py3k and sys.version_info[1] >= 2:
+ from html import escape
+else:
+ from cgi import escape
+
+# hrule styles
+FRAME = 0
+ALL = 1
+NONE = 2
+HEADER = 3
+
+# Table styles
+DEFAULT = 10
+MSWORD_FRIENDLY = 11
+PLAIN_COLUMNS = 12
+RANDOM = 20
+
+_re = re.compile("\033\[[0-9;]*m")
+
+def _get_size(text):
+ lines = text.split("\n")
+ height = len(lines)
+ width = max([_str_block_width(line) for line in lines])
+ return (width, height)
+
+class PrettyTable(object):
+
+ def __init__(self, field_names=None, **kwargs):
+
+ """Return a new PrettyTable instance
+
+ Arguments:
+
+ encoding - Unicode encoding scheme used to decode any encoded input
+ field_names - list or tuple of field names
+ fields - list or tuple of field names to include in displays
+ start - index of first data row to include in output
+ end - index of last data row to include in output PLUS ONE (list slice style)
+ header - print a header showing field names (True or False)
+ header_style - stylisation to apply to field names in header ("cap", "title", "upper", "lower" or None)
+ border - print a border around the table (True or False)
+ hrules - controls printing of horizontal rules after rows. Allowed values: FRAME, HEADER, ALL, NONE
+ vrules - controls printing of vertical rules between columns. Allowed values: FRAME, ALL, NONE
+ int_format - controls formatting of integer data
+ float_format - controls formatting of floating point data
+ padding_width - number of spaces on either side of column data (only used if left and right paddings are None)
+ left_padding_width - number of spaces on left hand side of column data
+ right_padding_width - number of spaces on right hand side of column data
+ vertical_char - single character string used to draw vertical lines
+ horizontal_char - single character string used to draw horizontal lines
+ junction_char - single character string used to draw line junctions
+ sortby - name of field to sort rows by
+ sort_key - sorting key function, applied to data points before sorting
+ valign - default valign for each row (None, "t", "m" or "b")
+ reversesort - True or False to sort in descending or ascending order"""
+
+ self.encoding = kwargs.get("encoding", "UTF-8")
+
+ # Data
+ self._field_names = []
+ self._align = {}
+ self._valign = {}
+ self._max_width = {}
+ self._rows = []
+ if field_names:
+ self.field_names = field_names
+ else:
+ self._widths = []
+
+ # Options
+ self._options = "start end fields header border sortby reversesort sort_key attributes format hrules vrules".split()
+ self._options.extend("int_format float_format padding_width left_padding_width right_padding_width".split())
+ self._options.extend("vertical_char horizontal_char junction_char header_style valign xhtml print_empty".split())
+ for option in self._options:
+ if option in kwargs:
+ self._validate_option(option, kwargs[option])
+ else:
+ kwargs[option] = None
+
+ self._start = kwargs["start"] or 0
+ self._end = kwargs["end"] or None
+ self._fields = kwargs["fields"] or None
+
+ if kwargs["header"] in (True, False):
+ self._header = kwargs["header"]
+ else:
+ self._header = True
+ self._header_style = kwargs["header_style"] or None
+ if kwargs["border"] in (True, False):
+ self._border = kwargs["border"]
+ else:
+ self._border = True
+ self._hrules = kwargs["hrules"] or FRAME
+ self._vrules = kwargs["vrules"] or ALL
+
+ self._sortby = kwargs["sortby"] or None
+ if kwargs["reversesort"] in (True, False):
+ self._reversesort = kwargs["reversesort"]
+ else:
+ self._reversesort = False
+ self._sort_key = kwargs["sort_key"] or (lambda x: x)
+
+ self._int_format = kwargs["int_format"] or {}
+ self._float_format = kwargs["float_format"] or {}
+ self._padding_width = kwargs["padding_width"] or 1
+ self._left_padding_width = kwargs["left_padding_width"] or None
+ self._right_padding_width = kwargs["right_padding_width"] or None
+
+ self._vertical_char = kwargs["vertical_char"] or self._unicode("|")
+ self._horizontal_char = kwargs["horizontal_char"] or self._unicode("-")
+ self._junction_char = kwargs["junction_char"] or self._unicode("+")
+
+ if kwargs["print_empty"] in (True, False):
+ self._print_empty = kwargs["print_empty"]
+ else:
+ self._print_empty = True
+ self._format = kwargs["format"] or False
+ self._xhtml = kwargs["xhtml"] or False
+ self._attributes = kwargs["attributes"] or {}
+
+ def _unicode(self, value):
+ if not isinstance(value, basestring):
+ value = str(value)
+ if not isinstance(value, unicode):
+ value = unicode(value, self.encoding, "strict")
+ return value
+
+ def _justify(self, text, width, align):
+ excess = width - _str_block_width(text)
+ if align == "l":
+ return text + excess * " "
+ elif align == "r":
+ return excess * " " + text
+ else:
+ if excess % 2:
+ # Uneven padding
+ # Put more space on right if text is of odd length...
+ if _str_block_width(text) % 2:
+ return (excess//2)*" " + text + (excess//2 + 1)*" "
+ # and more space on left if text is of even length
+ else:
+ return (excess//2 + 1)*" " + text + (excess//2)*" "
+ # Why distribute extra space this way? To match the behaviour of
+ # the inbuilt str.center() method.
+ else:
+ # Equal padding on either side
+ return (excess//2)*" " + text + (excess//2)*" "
+
+ def __getattr__(self, name):
+
+ if name == "rowcount":
+ return len(self._rows)
+ elif name == "colcount":
+ if self._field_names:
+ return len(self._field_names)
+ elif self._rows:
+ return len(self._rows[0])
+ else:
+ return 0
+ else:
+ raise AttributeError(name)
+
+ def __getitem__(self, index):
+
+ new = PrettyTable()
+ new.field_names = self.field_names
+ for attr in self._options:
+ setattr(new, "_"+attr, getattr(self, "_"+attr))
+ setattr(new, "_align", getattr(self, "_align"))
+ if isinstance(index, slice):
+ for row in self._rows[index]:
+ new.add_row(row)
+ elif isinstance(index, int):
+ new.add_row(self._rows[index])
+ else:
+ raise Exception("Index %s is invalid, must be an integer or slice" % str(index))
+ return new
+
+ if py3k:
+ def __str__(self):
+ return self.__unicode__()
+ else:
+ def __str__(self):
+ return self.__unicode__().encode(self.encoding)
+
+ def __unicode__(self):
+ return self.get_string()
+
+ ##############################
+ # ATTRIBUTE VALIDATORS #
+ ##############################
+
+ # The method _validate_option is all that should be used elsewhere in the code base to validate options.
+ # It will call the appropriate validation method for that option. The individual validation methods should
+ # never need to be called directly (although nothing bad will happen if they *are*).
+ # Validation happens in TWO places.
+ # Firstly, in the property setters defined in the ATTRIBUTE MANAGMENT section.
+ # Secondly, in the _get_options method, where keyword arguments are mixed with persistent settings
+
+ def _validate_option(self, option, val):
+ if option in ("field_names"):
+ self._validate_field_names(val)
+ elif option in ("start", "end", "max_width", "padding_width", "left_padding_width", "right_padding_width", "format"):
+ self._validate_nonnegative_int(option, val)
+ elif option in ("sortby"):
+ self._validate_field_name(option, val)
+ elif option in ("sort_key"):
+ self._validate_function(option, val)
+ elif option in ("hrules"):
+ self._validate_hrules(option, val)
+ elif option in ("vrules"):
+ self._validate_vrules(option, val)
+ elif option in ("fields"):
+ self._validate_all_field_names(option, val)
+ elif option in ("header", "border", "reversesort", "xhtml", "print_empty"):
+ self._validate_true_or_false(option, val)
+ elif option in ("header_style"):
+ self._validate_header_style(val)
+ elif option in ("int_format"):
+ self._validate_int_format(option, val)
+ elif option in ("float_format"):
+ self._validate_float_format(option, val)
+ elif option in ("vertical_char", "horizontal_char", "junction_char"):
+ self._validate_single_char(option, val)
+ elif option in ("attributes"):
+ self._validate_attributes(option, val)
+ else:
+ raise Exception("Unrecognised option: %s!" % option)
+
+ def _validate_field_names(self, val):
+ # Check for appropriate length
+ if self._field_names:
+ try:
+ assert len(val) == len(self._field_names)
+ except AssertionError:
+ raise Exception("Field name list has incorrect number of values, (actual) %d!=%d (expected)" % (len(val), len(self._field_names)))
+ if self._rows:
+ try:
+ assert len(val) == len(self._rows[0])
+ except AssertionError:
+ raise Exception("Field name list has incorrect number of values, (actual) %d!=%d (expected)" % (len(val), len(self._rows[0])))
+ # Check for uniqueness
+ try:
+ assert len(val) == len(set(val))
+ except AssertionError:
+ raise Exception("Field names must be unique!")
+
+ def _validate_header_style(self, val):
+ try:
+ assert val in ("cap", "title", "upper", "lower", None)
+ except AssertionError:
+ raise Exception("Invalid header style, use cap, title, upper, lower or None!")
+
+ def _validate_align(self, val):
+ try:
+ assert val in ["l","c","r"]
+ except AssertionError:
+ raise Exception("Alignment %s is invalid, use l, c or r!" % val)
+
+ def _validate_valign(self, val):
+ try:
+ assert val in ["t","m","b",None]
+ except AssertionError:
+ raise Exception("Alignment %s is invalid, use t, m, b or None!" % val)
+
+ def _validate_nonnegative_int(self, name, val):
+ try:
+ assert int(val) >= 0
+ except AssertionError:
+ raise Exception("Invalid value for %s: %s!" % (name, self._unicode(val)))
+
+ def _validate_true_or_false(self, name, val):
+ try:
+ assert val in (True, False)
+ except AssertionError:
+ raise Exception("Invalid value for %s! Must be True or False." % name)
+
+ def _validate_int_format(self, name, val):
+ if val == "":
+ return
+ try:
+ assert type(val) in (str, unicode)
+ assert val.isdigit()
+ except AssertionError:
+ raise Exception("Invalid value for %s! Must be an integer format string." % name)
+
+ def _validate_float_format(self, name, val):
+ if val == "":
+ return
+ try:
+ assert type(val) in (str, unicode)
+ assert "." in val
+ bits = val.split(".")
+ assert len(bits) <= 2
+ assert bits[0] == "" or bits[0].isdigit()
+ assert bits[1] == "" or bits[1].isdigit()
+ except AssertionError:
+ raise Exception("Invalid value for %s! Must be a float format string." % name)
+
+ def _validate_function(self, name, val):
+ try:
+ assert hasattr(val, "__call__")
+ except AssertionError:
+ raise Exception("Invalid value for %s! Must be a function." % name)
+
+ def _validate_hrules(self, name, val):
+ try:
+ assert val in (ALL, FRAME, HEADER, NONE)
+ except AssertionError:
+ raise Exception("Invalid value for %s! Must be ALL, FRAME, HEADER or NONE." % name)
+
+ def _validate_vrules(self, name, val):
+ try:
+ assert val in (ALL, FRAME, NONE)
+ except AssertionError:
+ raise Exception("Invalid value for %s! Must be ALL, FRAME, or NONE." % name)
+
+ def _validate_field_name(self, name, val):
+ try:
+ assert (val in self._field_names) or (val is None)
+ except AssertionError:
+ raise Exception("Invalid field name: %s!" % val)
+
+ def _validate_all_field_names(self, name, val):
+ try:
+ for x in val:
+ self._validate_field_name(name, x)
+ except AssertionError:
+ raise Exception("fields must be a sequence of field names!")
+
+ def _validate_single_char(self, name, val):
+ try:
+ assert _str_block_width(val) == 1
+ except AssertionError:
+ raise Exception("Invalid value for %s! Must be a string of length 1." % name)
+
+ def _validate_attributes(self, name, val):
+ try:
+ assert isinstance(val, dict)
+ except AssertionError:
+ raise Exception("attributes must be a dictionary of name/value pairs!")
+
+ ##############################
+ # ATTRIBUTE MANAGEMENT #
+ ##############################
+
+ def _get_field_names(self):
+ return self._field_names
+ """The names of the fields
+
+ Arguments:
+
+ fields - list or tuple of field names"""
+ def _set_field_names(self, val):
+ val = [self._unicode(x) for x in val]
+ self._validate_option("field_names", val)
+ if self._field_names:
+ old_names = self._field_names[:]
+ self._field_names = val
+ if self._align and old_names:
+ for old_name, new_name in zip(old_names, val):
+ self._align[new_name] = self._align[old_name]
+ for old_name in old_names:
+ if old_name not in self._align:
+ self._align.pop(old_name)
+ else:
+ for field in self._field_names:
+ self._align[field] = "c"
+ if self._valign and old_names:
+ for old_name, new_name in zip(old_names, val):
+ self._valign[new_name] = self._valign[old_name]
+ for old_name in old_names:
+ if old_name not in self._valign:
+ self._valign.pop(old_name)
+ else:
+ for field in self._field_names:
+ self._valign[field] = "t"
+ field_names = property(_get_field_names, _set_field_names)
+
+ def _get_align(self):
+ return self._align
+ def _set_align(self, val):
+ self._validate_align(val)
+ for field in self._field_names:
+ self._align[field] = val
+ align = property(_get_align, _set_align)
+
+ def _get_valign(self):
+ return self._valign
+ def _set_valign(self, val):
+ self._validate_valign(val)
+ for field in self._field_names:
+ self._valign[field] = val
+ valign = property(_get_valign, _set_valign)
+
+ def _get_max_width(self):
+ return self._max_width
+ def _set_max_width(self, val):
+ self._validate_option("max_width", val)
+ for field in self._field_names:
+ self._max_width[field] = val
+ max_width = property(_get_max_width, _set_max_width)
+
+ def _get_fields(self):
+ """List or tuple of field names to include in displays
+
+ Arguments:
+
+ fields - list or tuple of field names to include in displays"""
+ return self._fields
+ def _set_fields(self, val):
+ self._validate_option("fields", val)
+ self._fields = val
+ fields = property(_get_fields, _set_fields)
+
+ def _get_start(self):
+ """Start index of the range of rows to print
+
+ Arguments:
+
+ start - index of first data row to include in output"""
+ return self._start
+
+ def _set_start(self, val):
+ self._validate_option("start", val)
+ self._start = val
+ start = property(_get_start, _set_start)
+
+ def _get_end(self):
+ """End index of the range of rows to print
+
+ Arguments:
+
+ end - index of last data row to include in output PLUS ONE (list slice style)"""
+ return self._end
+ def _set_end(self, val):
+ self._validate_option("end", val)
+ self._end = val
+ end = property(_get_end, _set_end)
+
+ def _get_sortby(self):
+ """Name of field by which to sort rows
+
+ Arguments:
+
+ sortby - field name to sort by"""
+ return self._sortby
+ def _set_sortby(self, val):
+ self._validate_option("sortby", val)
+ self._sortby = val
+ sortby = property(_get_sortby, _set_sortby)
+
+ def _get_reversesort(self):
+ """Controls direction of sorting (ascending vs descending)
+
+ Arguments:
+
+ reveresort - set to True to sort by descending order, or False to sort by ascending order"""
+ return self._reversesort
+ def _set_reversesort(self, val):
+ self._validate_option("reversesort", val)
+ self._reversesort = val
+ reversesort = property(_get_reversesort, _set_reversesort)
+
+ def _get_sort_key(self):
+ """Sorting key function, applied to data points before sorting
+
+ Arguments:
+
+ sort_key - a function which takes one argument and returns something to be sorted"""
+ return self._sort_key
+ def _set_sort_key(self, val):
+ self._validate_option("sort_key", val)
+ self._sort_key = val
+ sort_key = property(_get_sort_key, _set_sort_key)
+
+ def _get_header(self):
+ """Controls printing of table header with field names
+
+ Arguments:
+
+ header - print a header showing field names (True or False)"""
+ return self._header
+ def _set_header(self, val):
+ self._validate_option("header", val)
+ self._header = val
+ header = property(_get_header, _set_header)
+
+ def _get_header_style(self):
+ """Controls stylisation applied to field names in header
+
+ Arguments:
+
+ header_style - stylisation to apply to field names in header ("cap", "title", "upper", "lower" or None)"""
+ return self._header_style
+ def _set_header_style(self, val):
+ self._validate_header_style(val)
+ self._header_style = val
+ header_style = property(_get_header_style, _set_header_style)
+
+ def _get_border(self):
+ """Controls printing of border around table
+
+ Arguments:
+
+ border - print a border around the table (True or False)"""
+ return self._border
+ def _set_border(self, val):
+ self._validate_option("border", val)
+ self._border = val
+ border = property(_get_border, _set_border)
+
+ def _get_hrules(self):
+ """Controls printing of horizontal rules after rows
+
+ Arguments:
+
+ hrules - horizontal rules style. Allowed values: FRAME, ALL, HEADER, NONE"""
+ return self._hrules
+ def _set_hrules(self, val):
+ self._validate_option("hrules", val)
+ self._hrules = val
+ hrules = property(_get_hrules, _set_hrules)
+
+ def _get_vrules(self):
+ """Controls printing of vertical rules between columns
+
+ Arguments:
+
+ vrules - vertical rules style. Allowed values: FRAME, ALL, NONE"""
+ return self._vrules
+ def _set_vrules(self, val):
+ self._validate_option("vrules", val)
+ self._vrules = val
+ vrules = property(_get_vrules, _set_vrules)
+
+ def _get_int_format(self):
+ """Controls formatting of integer data
+ Arguments:
+
+ int_format - integer format string"""
+ return self._int_format
+ def _set_int_format(self, val):
+# self._validate_option("int_format", val)
+ for field in self._field_names:
+ self._int_format[field] = val
+ int_format = property(_get_int_format, _set_int_format)
+
+ def _get_float_format(self):
+ """Controls formatting of floating point data
+ Arguments:
+
+ float_format - floating point format string"""
+ return self._float_format
+ def _set_float_format(self, val):
+# self._validate_option("float_format", val)
+ for field in self._field_names:
+ self._float_format[field] = val
+ float_format = property(_get_float_format, _set_float_format)
+
+ def _get_padding_width(self):
+ """The number of empty spaces between a column's edge and its content
+
+ Arguments:
+
+ padding_width - number of spaces, must be a positive integer"""
+ return self._padding_width
+ def _set_padding_width(self, val):
+ self._validate_option("padding_width", val)
+ self._padding_width = val
+ padding_width = property(_get_padding_width, _set_padding_width)
+
+ def _get_left_padding_width(self):
+ """The number of empty spaces between a column's left edge and its content
+
+ Arguments:
+
+ left_padding - number of spaces, must be a positive integer"""
+ return self._left_padding_width
+ def _set_left_padding_width(self, val):
+ self._validate_option("left_padding_width", val)
+ self._left_padding_width = val
+ left_padding_width = property(_get_left_padding_width, _set_left_padding_width)
+
+ def _get_right_padding_width(self):
+ """The number of empty spaces between a column's right edge and its content
+
+ Arguments:
+
+ right_padding - number of spaces, must be a positive integer"""
+ return self._right_padding_width
+ def _set_right_padding_width(self, val):
+ self._validate_option("right_padding_width", val)
+ self._right_padding_width = val
+ right_padding_width = property(_get_right_padding_width, _set_right_padding_width)
+
+ def _get_vertical_char(self):
+ """The charcter used when printing table borders to draw vertical lines
+
+ Arguments:
+
+ vertical_char - single character string used to draw vertical lines"""
+ return self._vertical_char
+ def _set_vertical_char(self, val):
+ val = self._unicode(val)
+ self._validate_option("vertical_char", val)
+ self._vertical_char = val
+ vertical_char = property(_get_vertical_char, _set_vertical_char)
+
+ def _get_horizontal_char(self):
+ """The charcter used when printing table borders to draw horizontal lines
+
+ Arguments:
+
+ horizontal_char - single character string used to draw horizontal lines"""
+ return self._horizontal_char
+ def _set_horizontal_char(self, val):
+ val = self._unicode(val)
+ self._validate_option("horizontal_char", val)
+ self._horizontal_char = val
+ horizontal_char = property(_get_horizontal_char, _set_horizontal_char)
+
+ def _get_junction_char(self):
+ """The charcter used when printing table borders to draw line junctions
+
+ Arguments:
+
+ junction_char - single character string used to draw line junctions"""
+ return self._junction_char
+ def _set_junction_char(self, val):
+ val = self._unicode(val)
+ self._validate_option("vertical_char", val)
+ self._junction_char = val
+ junction_char = property(_get_junction_char, _set_junction_char)
+
+ def _get_format(self):
+ """Controls whether or not HTML tables are formatted to match styling options
+
+ Arguments:
+
+ format - True or False"""
+ return self._format
+ def _set_format(self, val):
+ self._validate_option("format", val)
+ self._format = val
+ format = property(_get_format, _set_format)
+
+ def _get_print_empty(self):
+ """Controls whether or not empty tables produce a header and frame or just an empty string
+
+ Arguments:
+
+ print_empty - True or False"""
+ return self._print_empty
+ def _set_print_empty(self, val):
+ self._validate_option("print_empty", val)
+ self._print_empty = val
+ print_empty = property(_get_print_empty, _set_print_empty)
+
+ def _get_attributes(self):
+ """A dictionary of HTML attribute name/value pairs to be included in the <table> tag when printing HTML
+
+ Arguments:
+
+ attributes - dictionary of attributes"""
+ return self._attributes
+ def _set_attributes(self, val):
+ self._validate_option("attributes", val)
+ self._attributes = val
+ attributes = property(_get_attributes, _set_attributes)
+
+ ##############################
+ # OPTION MIXER #
+ ##############################
+
+ def _get_options(self, kwargs):
+
+ options = {}
+ for option in self._options:
+ if option in kwargs:
+ self._validate_option(option, kwargs[option])
+ options[option] = kwargs[option]
+ else:
+ options[option] = getattr(self, "_"+option)
+ return options
+
+ ##############################
+ # PRESET STYLE LOGIC #
+ ##############################
+
+ def set_style(self, style):
+
+ if style == DEFAULT:
+ self._set_default_style()
+ elif style == MSWORD_FRIENDLY:
+ self._set_msword_style()
+ elif style == PLAIN_COLUMNS:
+ self._set_columns_style()
+ elif style == RANDOM:
+ self._set_random_style()
+ else:
+ raise Exception("Invalid pre-set style!")
+
+ def _set_default_style(self):
+
+ self.header = True
+ self.border = True
+ self._hrules = FRAME
+ self._vrules = ALL
+ self.padding_width = 1
+ self.left_padding_width = 1
+ self.right_padding_width = 1
+ self.vertical_char = "|"
+ self.horizontal_char = "-"
+ self.junction_char = "+"
+
+ def _set_msword_style(self):
+
+ self.header = True
+ self.border = True
+ self._hrules = NONE
+ self.padding_width = 1
+ self.left_padding_width = 1
+ self.right_padding_width = 1
+ self.vertical_char = "|"
+
+ def _set_columns_style(self):
+
+ self.header = True
+ self.border = False
+ self.padding_width = 1
+ self.left_padding_width = 0
+ self.right_padding_width = 8
+
+ def _set_random_style(self):
+
+ # Just for fun!
+ self.header = random.choice((True, False))
+ self.border = random.choice((True, False))
+ self._hrules = random.choice((ALL, FRAME, HEADER, NONE))
+ self._vrules = random.choice((ALL, FRAME, NONE))
+ self.left_padding_width = random.randint(0,5)
+ self.right_padding_width = random.randint(0,5)
+ self.vertical_char = random.choice("~!@#$%^&*()_+|-=\{}[];':\",./;<>?")
+ self.horizontal_char = random.choice("~!@#$%^&*()_+|-=\{}[];':\",./;<>?")
+ self.junction_char = random.choice("~!@#$%^&*()_+|-=\{}[];':\",./;<>?")
+
+ ##############################
+ # DATA INPUT METHODS #
+ ##############################
+
+ def add_row(self, row):
+
+ """Add a row to the table
+
+ Arguments:
+
+ row - row of data, should be a list with as many elements as the table
+ has fields"""
+
+ if self._field_names and len(row) != len(self._field_names):
+ raise Exception("Row has incorrect number of values, (actual) %d!=%d (expected)" %(len(row),len(self._field_names)))
+ if not self._field_names:
+ self.field_names = [("Field %d" % (n+1)) for n in range(0,len(row))]
+ self._rows.append(list(row))
+
+ def del_row(self, row_index):
+
+ """Delete a row to the table
+
+ Arguments:
+
+ row_index - The index of the row you want to delete. Indexing starts at 0."""
+
+ if row_index > len(self._rows)-1:
+ raise Exception("Cant delete row at index %d, table only has %d rows!" % (row_index, len(self._rows)))
+ del self._rows[row_index]
+
+ def add_column(self, fieldname, column, align="c", valign="t"):
+
+ """Add a column to the table.
+
+ Arguments:
+
+ fieldname - name of the field to contain the new column of data
+ column - column of data, should be a list with as many elements as the
+ table has rows
+ align - desired alignment for this column - "l" for left, "c" for centre and "r" for right
+ valign - desired vertical alignment for new columns - "t" for top, "m" for middle and "b" for bottom"""
+
+ if len(self._rows) in (0, len(column)):
+ self._validate_align(align)
+ self._validate_valign(valign)
+ self._field_names.append(fieldname)
+ self._align[fieldname] = align
+ self._valign[fieldname] = valign
+ for i in range(0, len(column)):
+ if len(self._rows) < i+1:
+ self._rows.append([])
+ self._rows[i].append(column[i])
+ else:
+ raise Exception("Column length %d does not match number of rows %d!" % (len(column), len(self._rows)))
+
+ def clear_rows(self):
+
+ """Delete all rows from the table but keep the current field names"""
+
+ self._rows = []
+
+ def clear(self):
+
+ """Delete all rows and field names from the table, maintaining nothing but styling options"""
+
+ self._rows = []
+ self._field_names = []
+ self._widths = []
+
+ ##############################
+ # MISC PUBLIC METHODS #
+ ##############################
+
+ def copy(self):
+ return copy.deepcopy(self)
+
+ ##############################
+ # MISC PRIVATE METHODS #
+ ##############################
+
+ def _format_value(self, field, value):
+ if isinstance(value, int) and field in self._int_format:
+ value = self._unicode(("%%%sd" % self._int_format[field]) % value)
+ elif isinstance(value, float) and field in self._float_format:
+ value = self._unicode(("%%%sf" % self._float_format[field]) % value)
+ return self._unicode(value)
+
+ def _compute_widths(self, rows, options):
+ if options["header"]:
+ widths = [_get_size(field)[0] for field in self._field_names]
+ else:
+ widths = len(self.field_names) * [0]
+ for row in rows:
+ for index, value in enumerate(row):
+ fieldname = self.field_names[index]
+ if fieldname in self.max_width:
+ widths[index] = max(widths[index], min(_get_size(value)[0], self.max_width[fieldname]))
+ else:
+ widths[index] = max(widths[index], _get_size(value)[0])
+ self._widths = widths
+
+ def _get_padding_widths(self, options):
+
+ if options["left_padding_width"] is not None:
+ lpad = options["left_padding_width"]
+ else:
+ lpad = options["padding_width"]
+ if options["right_padding_width"] is not None:
+ rpad = options["right_padding_width"]
+ else:
+ rpad = options["padding_width"]
+ return lpad, rpad
+
+ def _get_rows(self, options):
+ """Return only those data rows that should be printed, based on slicing and sorting.
+
+ Arguments:
+
+ options - dictionary of option settings."""
+
+ # Make a copy of only those rows in the slice range
+ rows = copy.deepcopy(self._rows[options["start"]:options["end"]])
+ # Sort if necessary
+ if options["sortby"]:
+ sortindex = self._field_names.index(options["sortby"])
+ # Decorate
+ rows = [[row[sortindex]]+row for row in rows]
+ # Sort
+ rows.sort(reverse=options["reversesort"], key=options["sort_key"])
+ # Undecorate
+ rows = [row[1:] for row in rows]
+ return rows
+
+ def _format_row(self, row, options):
+ return [self._format_value(field, value) for (field, value) in zip(self._field_names, row)]
+
+ def _format_rows(self, rows, options):
+ return [self._format_row(row, options) for row in rows]
+
+ ##############################
+ # PLAIN TEXT STRING METHODS #
+ ##############################
+
+ def get_string(self, **kwargs):
+
+ """Return string representation of table in current state.
+
+ Arguments:
+
+ start - index of first data row to include in output
+ end - index of last data row to include in output PLUS ONE (list slice style)
+ fields - names of fields (columns) to include
+ header - print a header showing field names (True or False)
+ border - print a border around the table (True or False)
+ hrules - controls printing of horizontal rules after rows. Allowed values: ALL, FRAME, HEADER, NONE
+ vrules - controls printing of vertical rules between columns. Allowed values: FRAME, ALL, NONE
+ int_format - controls formatting of integer data
+ float_format - controls formatting of floating point data
+ padding_width - number of spaces on either side of column data (only used if left and right paddings are None)
+ left_padding_width - number of spaces on left hand side of column data
+ right_padding_width - number of spaces on right hand side of column data
+ vertical_char - single character string used to draw vertical lines
+ horizontal_char - single character string used to draw horizontal lines
+ junction_char - single character string used to draw line junctions
+ sortby - name of field to sort rows by
+ sort_key - sorting key function, applied to data points before sorting
+ reversesort - True or False to sort in descending or ascending order
+ print empty - if True, stringify just the header for an empty table, if False return an empty string """
+
+ options = self._get_options(kwargs)
+
+ lines = []
+
+ # Don't think too hard about an empty table
+ # Is this the desired behaviour? Maybe we should still print the header?
+ if self.rowcount == 0 and (not options["print_empty"] or not options["border"]):
+ return ""
+
+ # Get the rows we need to print, taking into account slicing, sorting, etc.
+ rows = self._get_rows(options)
+
+ # Turn all data in all rows into Unicode, formatted as desired
+ formatted_rows = self._format_rows(rows, options)
+
+ # Compute column widths
+ self._compute_widths(formatted_rows, options)
+
+ # Add header or top of border
+ self._hrule = self._stringify_hrule(options)
+ if options["header"]:
+ lines.append(self._stringify_header(options))
+ elif options["border"] and options["hrules"] in (ALL, FRAME):
+ lines.append(self._hrule)
+
+ # Add rows
+ for row in formatted_rows:
+ lines.append(self._stringify_row(row, options))
+
+ # Add bottom of border
+ if options["border"] and options["hrules"] == FRAME:
+ lines.append(self._hrule)
+
+ return self._unicode("\n").join(lines)
+
+ def _stringify_hrule(self, options):
+
+ if not options["border"]:
+ return ""
+ lpad, rpad = self._get_padding_widths(options)
+ if options['vrules'] in (ALL, FRAME):
+ bits = [options["junction_char"]]
+ else:
+ bits = [options["horizontal_char"]]
+ # For tables with no data or fieldnames
+ if not self._field_names:
+ bits.append(options["junction_char"])
+ return "".join(bits)
+ for field, width in zip(self._field_names, self._widths):
+ if options["fields"] and field not in options["fields"]:
+ continue
+ bits.append((width+lpad+rpad)*options["horizontal_char"])
+ if options['vrules'] == ALL:
+ bits.append(options["junction_char"])
+ else:
+ bits.append(options["horizontal_char"])
+ if options["vrules"] == FRAME:
+ bits.pop()
+ bits.append(options["junction_char"])
+ return "".join(bits)
+
+ def _stringify_header(self, options):
+
+ bits = []
+ lpad, rpad = self._get_padding_widths(options)
+ if options["border"]:
+ if options["hrules"] in (ALL, FRAME):
+ bits.append(self._hrule)
+ bits.append("\n")
+ if options["vrules"] in (ALL, FRAME):
+ bits.append(options["vertical_char"])
+ else:
+ bits.append(" ")
+ # For tables with no data or field names
+ if not self._field_names:
+ if options["vrules"] in (ALL, FRAME):
+ bits.append(options["vertical_char"])
+ else:
+ bits.append(" ")
+ for field, width, in zip(self._field_names, self._widths):
+ if options["fields"] and field not in options["fields"]:
+ continue
+ if self._header_style == "cap":
+ fieldname = field.capitalize()
+ elif self._header_style == "title":
+ fieldname = field.title()
+ elif self._header_style == "upper":
+ fieldname = field.upper()
+ elif self._header_style == "lower":
+ fieldname = field.lower()
+ else:
+ fieldname = field
+ bits.append(" " * lpad + self._justify(fieldname, width, self._align[field]) + " " * rpad)
+ if options["border"]:
+ if options["vrules"] == ALL:
+ bits.append(options["vertical_char"])
+ else:
+ bits.append(" ")
+ # If vrules is FRAME, then we just appended a space at the end
+ # of the last field, when we really want a vertical character
+ if options["border"] and options["vrules"] == FRAME:
+ bits.pop()
+ bits.append(options["vertical_char"])
+ if options["border"] and options["hrules"] != NONE:
+ bits.append("\n")
+ bits.append(self._hrule)
+ return "".join(bits)
+
+ def _stringify_row(self, row, options):
+
+ for index, field, value, width, in zip(range(0,len(row)), self._field_names, row, self._widths):
+ # Enforce max widths
+ lines = value.split("\n")
+ new_lines = []
+ for line in lines:
+ if _str_block_width(line) > width:
+ line = textwrap.fill(line, width)
+ new_lines.append(line)
+ lines = new_lines
+ value = "\n".join(lines)
+ row[index] = value
+
+ row_height = 0
+ for c in row:
+ h = _get_size(c)[1]
+ if h > row_height:
+ row_height = h
+
+ bits = []
+ lpad, rpad = self._get_padding_widths(options)
+ for y in range(0, row_height):
+ bits.append([])
+ if options["border"]:
+ if options["vrules"] in (ALL, FRAME):
+ bits[y].append(self.vertical_char)
+ else:
+ bits[y].append(" ")
+
+ for field, value, width, in zip(self._field_names, row, self._widths):
+
+ valign = self._valign[field]
+ lines = value.split("\n")
+ dHeight = row_height - len(lines)
+ if dHeight:
+ if valign == "m":
+ lines = [""] * int(dHeight / 2) + lines + [""] * (dHeight - int(dHeight / 2))
+ elif valign == "b":
+ lines = [""] * dHeight + lines
+ else:
+ lines = lines + [""] * dHeight
+
+ y = 0
+ for l in lines:
+ if options["fields"] and field not in options["fields"]:
+ continue
+
+ bits[y].append(" " * lpad + self._justify(l, width, self._align[field]) + " " * rpad)
+ if options["border"]:
+ if options["vrules"] == ALL:
+ bits[y].append(self.vertical_char)
+ else:
+ bits[y].append(" ")
+ y += 1
+
+ # If vrules is FRAME, then we just appended a space at the end
+ # of the last field, when we really want a vertical character
+ for y in range(0, row_height):
+ if options["border"] and options["vrules"] == FRAME:
+ bits[y].pop()
+ bits[y].append(options["vertical_char"])
+
+ if options["border"] and options["hrules"]== ALL:
+ bits[row_height-1].append("\n")
+ bits[row_height-1].append(self._hrule)
+
+ for y in range(0, row_height):
+ bits[y] = "".join(bits[y])
+
+ return "\n".join(bits)
+
+ ##############################
+ # HTML STRING METHODS #
+ ##############################
+
+ def get_html_string(self, **kwargs):
+
+ """Return string representation of HTML formatted version of table in current state.
+
+ Arguments:
+
+ start - index of first data row to include in output
+ end - index of last data row to include in output PLUS ONE (list slice style)
+ fields - names of fields (columns) to include
+ header - print a header showing field names (True or False)
+ border - print a border around the table (True or False)
+ hrules - controls printing of horizontal rules after rows. Allowed values: ALL, FRAME, HEADER, NONE
+ vrules - controls printing of vertical rules between columns. Allowed values: FRAME, ALL, NONE
+ int_format - controls formatting of integer data
+ float_format - controls formatting of floating point data
+ padding_width - number of spaces on either side of column data (only used if left and right paddings are None)
+ left_padding_width - number of spaces on left hand side of column data
+ right_padding_width - number of spaces on right hand side of column data
+ sortby - name of field to sort rows by
+ sort_key - sorting key function, applied to data points before sorting
+ attributes - dictionary of name/value pairs to include as HTML attributes in the <table> tag
+ xhtml - print <br/> tags if True, <br> tags if false"""
+
+ options = self._get_options(kwargs)
+
+ if options["format"]:
+ string = self._get_formatted_html_string(options)
+ else:
+ string = self._get_simple_html_string(options)
+
+ return string
+
+ def _get_simple_html_string(self, options):
+
+ lines = []
+ if options["xhtml"]:
+ linebreak = "<br/>"
+ else:
+ linebreak = "<br>"
+
+ open_tag = []
+ open_tag.append("<table")
+ if options["attributes"]:
+ for attr_name in options["attributes"]:
+ open_tag.append(" %s=\"%s\"" % (attr_name, options["attributes"][attr_name]))
+ open_tag.append(">")
+ lines.append("".join(open_tag))
+
+ # Headers
+ if options["header"]:
+ lines.append(" <tr>")
+ for field in self._field_names:
+ if options["fields"] and field not in options["fields"]:
+ continue
+ lines.append(" <th>%s</th>" % escape(field).replace("\n", linebreak))
+ lines.append(" </tr>")
+
+ # Data
+ rows = self._get_rows(options)
+ formatted_rows = self._format_rows(rows, options)
+ for row in formatted_rows:
+ lines.append(" <tr>")
+ for field, datum in zip(self._field_names, row):
+ if options["fields"] and field not in options["fields"]:
+ continue
+ lines.append(" <td>%s</td>" % escape(datum).replace("\n", linebreak))
+ lines.append(" </tr>")
+
+ lines.append("</table>")
+
+ return self._unicode("\n").join(lines)
+
+ def _get_formatted_html_string(self, options):
+
+ lines = []
+ lpad, rpad = self._get_padding_widths(options)
+ if options["xhtml"]:
+ linebreak = "<br/>"
+ else:
+ linebreak = "<br>"
+
+ open_tag = []
+ open_tag.append("<table")
+ if options["border"]:
+ if options["hrules"] == ALL and options["vrules"] == ALL:
+ open_tag.append(" frame=\"box\" rules=\"all\"")
+ elif options["hrules"] == FRAME and options["vrules"] == FRAME:
+ open_tag.append(" frame=\"box\"")
+ elif options["hrules"] == FRAME and options["vrules"] == ALL:
+ open_tag.append(" frame=\"box\" rules=\"cols\"")
+ elif options["hrules"] == FRAME:
+ open_tag.append(" frame=\"hsides\"")
+ elif options["hrules"] == ALL:
+ open_tag.append(" frame=\"hsides\" rules=\"rows\"")
+ elif options["vrules"] == FRAME:
+ open_tag.append(" frame=\"vsides\"")
+ elif options["vrules"] == ALL:
+ open_tag.append(" frame=\"vsides\" rules=\"cols\"")
+ if options["attributes"]:
+ for attr_name in options["attributes"]:
+ open_tag.append(" %s=\"%s\"" % (attr_name, options["attributes"][attr_name]))
+ open_tag.append(">")
+ lines.append("".join(open_tag))
+
+ # Headers
+ if options["header"]:
+ lines.append(" <tr>")
+ for field in self._field_names:
+ if options["fields"] and field not in options["fields"]:
+ continue
+ lines.append(" <th style=\"padding-left: %dem; padding-right: %dem; text-align: center\">%s</th>" % (lpad, rpad, escape(field).replace("\n", linebreak)))
+ lines.append(" </tr>")
+
+ # Data
+ rows = self._get_rows(options)
+ formatted_rows = self._format_rows(rows, options)
+ aligns = []
+ valigns = []
+ for field in self._field_names:
+ aligns.append({ "l" : "left", "r" : "right", "c" : "center" }[self._align[field]])
+ valigns.append({"t" : "top", "m" : "middle", "b" : "bottom"}[self._valign[field]])
+ for row in formatted_rows:
+ lines.append(" <tr>")
+ for field, datum, align, valign in zip(self._field_names, row, aligns, valigns):
+ if options["fields"] and field not in options["fields"]:
+ continue
+ lines.append(" <td style=\"padding-left: %dem; padding-right: %dem; text-align: %s; vertical-align: %s\">%s</td>" % (lpad, rpad, align, valign, escape(datum).replace("\n", linebreak)))
+ lines.append(" </tr>")
+ lines.append("</table>")
+
+ return self._unicode("\n").join(lines)
+
+##############################
+# UNICODE WIDTH FUNCTIONS #
+##############################
+
+def _char_block_width(char):
+ # Basic Latin, which is probably the most common case
+ #if char in xrange(0x0021, 0x007e):
+ #if char >= 0x0021 and char <= 0x007e:
+ if 0x0021 <= char <= 0x007e:
+ return 1
+ # Chinese, Japanese, Korean (common)
+ if 0x4e00 <= char <= 0x9fff:
+ return 2
+ # Hangul
+ if 0xac00 <= char <= 0xd7af:
+ return 2
+ # Combining?
+ if unicodedata.combining(uni_chr(char)):
+ return 0
+ # Hiragana and Katakana
+ if 0x3040 <= char <= 0x309f or 0x30a0 <= char <= 0x30ff:
+ return 2
+ # Full-width Latin characters
+ if 0xff01 <= char <= 0xff60:
+ return 2
+ # CJK punctuation
+ if 0x3000 <= char <= 0x303e:
+ return 2
+ # Backspace and delete
+ if char in (0x0008, 0x007f):
+ return -1
+ # Other control characters
+ elif char in (0x0000, 0x001f):
+ return 0
+ # Take a guess
+ return 1
+
+def _str_block_width(val):
+
+ return sum(itermap(_char_block_width, itermap(ord, _re.sub("", val))))
+
+##############################
+# TABLE FACTORIES #
+##############################
+
+def from_csv(fp, field_names = None, **kwargs):
+
+ dialect = csv.Sniffer().sniff(fp.read(1024))
+ fp.seek(0)
+ reader = csv.reader(fp, dialect)
+
+ table = PrettyTable(**kwargs)
+ if field_names:
+ table.field_names = field_names
+ else:
+ if py3k:
+ table.field_names = [x.strip() for x in next(reader)]
+ else:
+ table.field_names = [x.strip() for x in reader.next()]
+
+ for row in reader:
+ table.add_row([x.strip() for x in row])
+
+ return table
+
+def from_db_cursor(cursor, **kwargs):
+
+ if cursor.description:
+ table = PrettyTable(**kwargs)
+ table.field_names = [col[0] for col in cursor.description]
+ for row in cursor.fetchall():
+ table.add_row(row)
+ return table
+
+class TableHandler(HTMLParser):
+
+ def __init__(self, **kwargs):
+ HTMLParser.__init__(self)
+ self.kwargs = kwargs
+ self.tables = []
+ self.last_row = []
+ self.rows = []
+ self.max_row_width = 0
+ self.active = None
+ self.last_content = ""
+ self.is_last_row_header = False
+
+ def handle_starttag(self,tag, attrs):
+ self.active = tag
+ if tag == "th":
+ self.is_last_row_header = True
+
+ def handle_endtag(self,tag):
+ if tag in ["th", "td"]:
+ stripped_content = self.last_content.strip()
+ self.last_row.append(stripped_content)
+ if tag == "tr":
+ self.rows.append(
+ (self.last_row, self.is_last_row_header))
+ self.max_row_width = max(self.max_row_width, len(self.last_row))
+ self.last_row = []
+ self.is_last_row_header = False
+ if tag == "table":
+ table = self.generate_table(self.rows)
+ self.tables.append(table)
+ self.rows = []
+ self.last_content = " "
+ self.active = None
+
+
+ def handle_data(self, data):
+ self.last_content += data
+
+ def generate_table(self, rows):
+ """
+ Generates from a list of rows a PrettyTable object.
+ """
+ table = PrettyTable(**self.kwargs)
+ for row in self.rows:
+ if len(row[0]) < self.max_row_width:
+ appends = self.max_row_width - len(row[0])
+ for i in range(1,appends):
+ row[0].append("-")
+
+ if row[1] == True:
+ self.make_fields_unique(row[0])
+ table.field_names = row[0]
+ else:
+ table.add_row(row[0])
+ return table
+
+ def make_fields_unique(self, fields):
+ """
+ iterates over the row and make each field unique
+ """
+ for i in range(0, len(fields)):
+ for j in range(i+1, len(fields)):
+ if fields[i] == fields[j]:
+ fields[j] += "'"
+
+def from_html(html_code, **kwargs):
+ """
+ Generates a list of PrettyTables from a string of HTML code. Each <table> in
+ the HTML becomes one PrettyTable object.
+ """
+
+ parser = TableHandler(**kwargs)
+ parser.feed(html_code)
+ return parser.tables
+
+def from_html_one(html_code, **kwargs):
+ """
+ Generates a PrettyTables from a string of HTML code which contains only a
+ single <table>
+ """
+
+ tables = from_html(html_code, **kwargs)
+ try:
+ assert len(tables) == 1
+ except AssertionError:
+ raise Exception("More than one <table> in provided HTML code! Use from_html instead.")
+ return tables[0]
+
+##############################
+# MAIN (TEST FUNCTION) #
+##############################
+
+def main():
+
+ x = PrettyTable(["City name", "Area", "Population", "Annual Rainfall"])
+ x.sortby = "Population"
+ x.reversesort = True
+ x.int_format["Area"] = "04d"
+ x.float_format = "6.1f"
+ x.align["City name"] = "l" # Left align city names
+ x.add_row(["Adelaide", 1295, 1158259, 600.5])
+ x.add_row(["Brisbane", 5905, 1857594, 1146.4])
+ x.add_row(["Darwin", 112, 120900, 1714.7])
+ x.add_row(["Hobart", 1357, 205556, 619.5])
+ x.add_row(["Sydney", 2058, 4336374, 1214.8])
+ x.add_row(["Melbourne", 1566, 3806092, 646.9])
+ x.add_row(["Perth", 5386, 1554769, 869.4])
+ print(x)
+
+if __name__ == "__main__":
+ main()