You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by zs...@apache.org on 2009/03/26 22:12:26 UTC

svn commit: r758857 [3/3] - in /hadoop/hive/trunk: ./ service/if/ service/lib/ service/lib/py/ service/lib/py/fb303/ service/lib/py/fb303_scripts/ service/lib/py/thrift/ service/lib/py/thrift/protocol/ service/lib/py/thrift/reflection/ service/lib/py/t...

Added: hadoop/hive/trunk/service/lib/py/thrift/transport/TTransport.py
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/service/lib/py/thrift/transport/TTransport.py?rev=758857&view=auto
==============================================================================
--- hadoop/hive/trunk/service/lib/py/thrift/transport/TTransport.py (added)
+++ hadoop/hive/trunk/service/lib/py/thrift/transport/TTransport.py Thu Mar 26 21:12:24 2009
@@ -0,0 +1,313 @@
+# Copyright (c) 2006- Facebook
+# Distributed under the Thrift Software License
+#
+# See accompanying file LICENSE or visit the Thrift site at:
+# http://developers.facebook.com/thrift/
+
+from cStringIO import StringIO
+from struct import pack,unpack
+from thrift.Thrift import TException
+
+class TTransportException(TException):
+
+  """Custom Transport Exception class"""
+
+  UNKNOWN = 0
+  NOT_OPEN = 1
+  ALREADY_OPEN = 2
+  TIMED_OUT = 3
+  END_OF_FILE = 4
+
+  def __init__(self, type=UNKNOWN, message=None):
+    TException.__init__(self, message)
+    self.type = type
+
+class TTransportBase:
+
+  """Base class for Thrift transport layer."""
+
+  def isOpen(self):
+    pass
+
+  def open(self):
+    pass
+
+  def close(self):
+    pass
+
+  def read(self, sz):
+    pass
+
+  def readAll(self, sz):
+    buff = ''
+    have = 0
+    while (have < sz):
+      chunk = self.read(sz-have)
+      have += len(chunk)
+      buff += chunk
+
+      if len(chunk) == 0:
+        raise EOFError()
+
+    return buff
+
+  def write(self, buf):
+    pass
+
+  def flush(self):
+    pass
+
+# This class should be thought of as an interface.
+class CReadableTransport:
+  """base class for transports that are readable from C"""
+
+  # TODO(dreiss): Think about changing this interface to allow us to use
+  #               a (Python, not c) StringIO instead, because it allows
+  #               you to write after reading.
+
+  # NOTE: This is a classic class, so properties will NOT work
+  #       correctly for setting.
+  @property
+  def cstringio_buf(self):
+    """A cStringIO buffer that contains the current chunk we are reading."""
+    pass
+
+  def cstringio_refill(self, partialread, reqlen):
+    """Refills cstringio_buf.
+
+    Returns the currently used buffer (which can but need not be the same as
+    the old cstringio_buf). partialread is what the C code has read from the
+    buffer, and should be inserted into the buffer before any more reads.  The
+    return value must be a new, not borrowed reference.  Something along the
+    lines of self._buf should be fine.
+
+    If reqlen bytes can't be read, throw EOFError.
+    """
+    pass
+
+class TServerTransportBase:
+
+  """Base class for Thrift server transports."""
+
+  def listen(self):
+    pass
+
+  def accept(self):
+    pass
+
+  def close(self):
+    pass
+
+class TTransportFactoryBase:
+
+  """Base class for a Transport Factory"""
+
+  def getTransport(self, trans):
+    return trans
+
+class TBufferedTransportFactory:
+
+  """Factory transport that builds buffered transports"""
+
+  def getTransport(self, trans):
+    buffered = TBufferedTransport(trans)
+    return buffered
+
+
+class TBufferedTransport(TTransportBase,CReadableTransport):
+
+  """Class that wraps another transport and buffers its I/O."""
+
+  DEFAULT_BUFFER = 4096
+
+  def __init__(self, trans):
+    self.__trans = trans
+    self.__wbuf = StringIO()
+    self.__rbuf = StringIO("")
+
+  def isOpen(self):
+    return self.__trans.isOpen()
+
+  def open(self):
+    return self.__trans.open()
+
+  def close(self):
+    return self.__trans.close()
+
+  def read(self, sz):
+    ret = self.__rbuf.read(sz)
+    if len(ret) != 0:
+      return ret
+
+    self.__rbuf = StringIO(self.__trans.read(max(sz, self.DEFAULT_BUFFER)))
+    return self.__rbuf.read(sz)
+
+  def write(self, buf):
+    self.__wbuf.write(buf)
+
+  def flush(self):
+    out = self.__wbuf.getvalue()
+    # reset wbuf before write/flush to preserve state on underlying failure
+    self.__wbuf = StringIO()
+    self.__trans.write(out)
+    self.__trans.flush()
+
+  # Implement the CReadableTransport interface.
+  @property
+  def cstringio_buf(self):
+    return self.__rbuf
+
+  def cstringio_refill(self, partialread, reqlen):
+    retstring = partialread
+    if reqlen < self.DEFAULT_BUFFER:
+      # try to make a read of as much as we can.
+      retstring += self.__trans.read(self.DEFAULT_BUFFER)
+
+    # but make sure we do read reqlen bytes.
+    if len(retstring) < reqlen:
+      retstring += self.__trans.readAll(reqlen - len(retstring))
+
+    self.__rbuf = StringIO(retstring)
+    return self.__rbuf
+
+class TMemoryBuffer(TTransportBase, CReadableTransport):
+  """Wraps a cStringIO object as a TTransport.
+
+  NOTE: Unlike the C++ version of this class, you cannot write to it
+        then immediately read from it.  If you want to read from a
+        TMemoryBuffer, you must either pass a string to the constructor.
+  TODO(dreiss): Make this work like the C++ version.
+  """
+
+  def __init__(self, value=None):
+    """value -- a value to read from for stringio
+
+    If value is set, this will be a transport for reading,
+    otherwise, it is for writing"""
+    if value is not None:
+      self._buffer = StringIO(value)
+    else:
+      self._buffer = StringIO()
+
+  def isOpen(self):
+    return not self._buffer.closed
+
+  def open(self):
+    pass
+
+  def close(self):
+    self._buffer.close()
+
+  def read(self, sz):
+    return self._buffer.read(sz)
+
+  def write(self, buf):
+    self._buffer.write(buf)
+
+  def flush(self):
+    pass
+
+  def getvalue(self):
+    return self._buffer.getvalue()
+
+  # Implement the CReadableTransport interface.
+  @property
+  def cstringio_buf(self):
+    return self._buffer
+
+  def cstringio_refill(self, partialread, reqlen):
+    # only one shot at reading...
+    raise EOFError()
+
+class TFramedTransportFactory:
+
+  """Factory transport that builds framed transports"""
+
+  def getTransport(self, trans):
+    framed = TFramedTransport(trans)
+    return framed
+
+
+class TFramedTransport(TTransportBase, CReadableTransport):
+
+  """Class that wraps another transport and frames its I/O when writing."""
+
+  def __init__(self, trans,):
+    self.__trans = trans
+    self.__rbuf = StringIO()
+    self.__wbuf = StringIO()
+
+  def isOpen(self):
+    return self.__trans.isOpen()
+
+  def open(self):
+    return self.__trans.open()
+
+  def close(self):
+    return self.__trans.close()
+
+  def read(self, sz):
+    ret = self.__rbuf.read(sz)
+    if len(ret) != 0:
+      return ret
+
+    self.readFrame()
+    return self.__rbuf.read(sz)
+
+  def readFrame(self):
+    buff = self.__trans.readAll(4)
+    sz, = unpack('!i', buff)
+    self.__rbuf = StringIO(self.__trans.readAll(sz))
+
+  def write(self, buf):
+    self.__wbuf.write(buf)
+
+  def flush(self):
+    wout = self.__wbuf.getvalue()
+    wsz = len(wout)
+    # reset wbuf before write/flush to preserve state on underlying failure
+    self.__wbuf = StringIO()
+    # N.B.: Doing this string concatenation is WAY cheaper than making
+    # two separate calls to the underlying socket object. Socket writes in
+    # Python turn out to be REALLY expensive, but it seems to do a pretty
+    # good job of managing string buffer operations without excessive copies
+    buf = pack("!i", wsz) + wout
+    self.__trans.write(buf)
+    self.__trans.flush()
+
+  # Implement the CReadableTransport interface.
+  @property
+  def cstringio_buf(self):
+    return self.__rbuf
+
+  def cstringio_refill(self, prefix, reqlen):
+    # self.__rbuf will already be empty here because fastbinary doesn't
+    # ask for a refill until the previous buffer is empty.  Therefore,
+    # we can start reading new frames immediately.
+    while len(prefix) < reqlen:
+      readFrame()
+      prefix += self.__rbuf.getvalue()
+    self.__rbuf = StringIO(prefix)
+    return self.__rbuf
+
+
+class TFileObjectTransport(TTransportBase):
+  """Wraps a file-like object to make it work as a Thrift transport."""
+
+  def __init__(self, fileobj):
+    self.fileobj = fileobj
+
+  def isOpen(self):
+    return True
+
+  def close(self):
+    self.fileobj.close()
+
+  def read(self, sz):
+    return self.fileobj.read(sz)
+
+  def write(self, buf):
+    self.fileobj.write(buf)
+
+  def flush(self):
+    self.fileobj.flush()

Added: hadoop/hive/trunk/service/lib/py/thrift/transport/TTwisted.py
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/service/lib/py/thrift/transport/TTwisted.py?rev=758857&view=auto
==============================================================================
--- hadoop/hive/trunk/service/lib/py/thrift/transport/TTwisted.py (added)
+++ hadoop/hive/trunk/service/lib/py/thrift/transport/TTwisted.py Thu Mar 26 21:12:24 2009
@@ -0,0 +1,157 @@
+from zope.interface import implements, Interface, Attribute
+from twisted.internet.protocol import Protocol, ServerFactory, ClientFactory, \
+    connectionDone
+from twisted.internet import defer
+from twisted.protocols import basic
+from twisted.python import log
+from thrift.transport import TTransport
+from cStringIO import StringIO
+
+
+class TMessageSenderTransport(TTransport.TTransportBase):
+
+    def __init__(self):
+        self.__wbuf = StringIO()
+
+    def write(self, buf):
+        self.__wbuf.write(buf)
+
+    def flush(self):
+        msg = self.__wbuf.getvalue()
+        self.__wbuf = StringIO()
+        self.sendMessage(msg)
+
+    def sendMessage(self, message):
+        raise NotImplementedError
+
+
+class TCallbackTransport(TMessageSenderTransport):
+
+    def __init__(self, func):
+        TMessageSenderTransport.__init__(self)
+        self.func = func
+
+    def sendMessage(self, message):
+        self.func(message)
+
+
+class ThriftClientProtocol(basic.Int32StringReceiver):
+
+    def __init__(self, client_class, iprot_factory, oprot_factory=None):
+        self._client_class = client_class
+        self._iprot_factory = iprot_factory
+        if oprot_factory is None:
+            self._oprot_factory = iprot_factory
+        else:
+            self._oprot_factory = oprot_factory
+
+        self.recv_map = {}
+        self.started = defer.Deferred()
+
+    def dispatch(self, msg):
+        self.sendString(msg)
+
+    def connectionMade(self):
+        tmo = TCallbackTransport(self.dispatch)
+        self.client = self._client_class(tmo, self._oprot_factory)
+        self.started.callback(self.client)
+
+    def connectionLost(self, reason=connectionDone):
+        for k,v in self.client._reqs.iteritems():
+            tex = TTransport.TTransportException(
+                type=TTransport.TTransportException.END_OF_FILE,
+                message='Connection closed')
+            v.errback(tex)
+
+    def stringReceived(self, frame):
+        tr = TTransport.TMemoryBuffer(frame)
+        iprot = self._iprot_factory.getProtocol(tr)
+        (fname, mtype, rseqid) = iprot.readMessageBegin()
+
+        try:
+            method = self.recv_map[fname]
+        except KeyError:
+            method = getattr(self.client, 'recv_' + fname)
+            self.recv_map[fname] = method
+
+        method(iprot, mtype, rseqid)
+
+
+class ThriftServerProtocol(basic.Int32StringReceiver):
+
+    def dispatch(self, msg):
+        self.sendString(msg)
+
+    def processError(self, error):
+        self.transport.loseConnection()
+
+    def processOk(self, _, tmo):
+        msg = tmo.getvalue()
+
+        if len(msg) > 0:
+            self.dispatch(msg)
+
+    def stringReceived(self, frame):
+        tmi = TTransport.TMemoryBuffer(frame)
+        tmo = TTransport.TMemoryBuffer()
+
+        iprot = self.factory.iprot_factory.getProtocol(tmi)
+        oprot = self.factory.oprot_factory.getProtocol(tmo)
+
+        d = self.factory.processor.process(iprot, oprot)
+        d.addCallbacks(self.processOk, self.processError,
+            callbackArgs=(tmo,))
+
+
+class IThriftServerFactory(Interface):
+
+    processor = Attribute("Thrift processor")
+
+    iprot_factory = Attribute("Input protocol factory")
+
+    oprot_factory = Attribute("Output protocol factory")
+
+
+class IThriftClientFactory(Interface):
+
+    client_class = Attribute("Thrift client class")
+
+    iprot_factory = Attribute("Input protocol factory")
+
+    oprot_factory = Attribute("Output protocol factory")
+
+
+class ThriftServerFactory(ServerFactory):
+
+    implements(IThriftServerFactory)
+
+    protocol = ThriftServerProtocol
+
+    def __init__(self, processor, iprot_factory, oprot_factory=None):
+        self.processor = processor
+        self.iprot_factory = iprot_factory
+        if oprot_factory is None:
+            self.oprot_factory = iprot_factory
+        else:
+            self.oprot_factory = oprot_factory
+
+
+class ThriftClientFactory(ClientFactory):
+
+    implements(IThriftClientFactory)
+
+    protocol = ThriftClientProtocol
+
+    def __init__(self, client_class, iprot_factory, oprot_factory=None):
+        self.client_class = client_class
+        self.iprot_factory = iprot_factory
+        if oprot_factory is None:
+            self.oprot_factory = iprot_factory
+        else:
+            self.oprot_factory = oprot_factory
+
+    def buildProtocol(self, addr):
+        p = self.protocol(self.client_class, self.iprot_factory,
+            self.oprot_factory)
+        p.factory = self
+        return p

Added: hadoop/hive/trunk/service/lib/py/thrift/transport/__init__.py
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/service/lib/py/thrift/transport/__init__.py?rev=758857&view=auto
==============================================================================
--- hadoop/hive/trunk/service/lib/py/thrift/transport/__init__.py (added)
+++ hadoop/hive/trunk/service/lib/py/thrift/transport/__init__.py Thu Mar 26 21:12:24 2009
@@ -0,0 +1,7 @@
+# Copyright (c) 2006- Facebook
+# Distributed under the Thrift Software License
+#
+# See accompanying file LICENSE or visit the Thrift site at:
+# http://developers.facebook.com/thrift/
+
+__all__ = ['TTransport', 'TSocket', 'THttpClient']

Modified: hadoop/hive/trunk/service/src/gen-py/hive_service/ThriftHive.py
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/service/src/gen-py/hive_service/ThriftHive.py?rev=758857&r1=758856&r2=758857&view=diff
==============================================================================
--- hadoop/hive/trunk/service/src/gen-py/hive_service/ThriftHive.py (original)
+++ hadoop/hive/trunk/service/src/gen-py/hive_service/ThriftHive.py Thu Mar 26 21:12:24 2009
@@ -314,10 +314,10 @@
     oprot.writeFieldStop()
     oprot.writeStructEnd()
 
-  def __str__(self):
+  def __str__(self): 
     return str(self.__dict__)
 
-  def __repr__(self):
+  def __repr__(self): 
     return repr(self.__dict__)
 
   def __eq__(self, other):
@@ -371,10 +371,10 @@
     oprot.writeFieldStop()
     oprot.writeStructEnd()
 
-  def __str__(self):
+  def __str__(self): 
     return str(self.__dict__)
 
-  def __repr__(self):
+  def __repr__(self): 
     return repr(self.__dict__)
 
   def __eq__(self, other):
@@ -413,10 +413,10 @@
     oprot.writeFieldStop()
     oprot.writeStructEnd()
 
-  def __str__(self):
+  def __str__(self): 
     return str(self.__dict__)
 
-  def __repr__(self):
+  def __repr__(self): 
     return repr(self.__dict__)
 
   def __eq__(self, other):
@@ -482,10 +482,10 @@
     oprot.writeFieldStop()
     oprot.writeStructEnd()
 
-  def __str__(self):
+  def __str__(self): 
     return str(self.__dict__)
 
-  def __repr__(self):
+  def __repr__(self): 
     return repr(self.__dict__)
 
   def __eq__(self, other):
@@ -538,10 +538,10 @@
     oprot.writeFieldStop()
     oprot.writeStructEnd()
 
-  def __str__(self):
+  def __str__(self): 
     return str(self.__dict__)
 
-  def __repr__(self):
+  def __repr__(self): 
     return repr(self.__dict__)
 
   def __eq__(self, other):
@@ -615,10 +615,10 @@
     oprot.writeFieldStop()
     oprot.writeStructEnd()
 
-  def __str__(self):
+  def __str__(self): 
     return str(self.__dict__)
 
-  def __repr__(self):
+  def __repr__(self): 
     return repr(self.__dict__)
 
   def __eq__(self, other):
@@ -657,10 +657,10 @@
     oprot.writeFieldStop()
     oprot.writeStructEnd()
 
-  def __str__(self):
+  def __str__(self): 
     return str(self.__dict__)
 
-  def __repr__(self):
+  def __repr__(self): 
     return repr(self.__dict__)
 
   def __eq__(self, other):
@@ -734,10 +734,10 @@
     oprot.writeFieldStop()
     oprot.writeStructEnd()
 
-  def __str__(self):
+  def __str__(self): 
     return str(self.__dict__)
 
-  def __repr__(self):
+  def __repr__(self): 
     return repr(self.__dict__)
 
   def __eq__(self, other):
@@ -776,10 +776,10 @@
     oprot.writeFieldStop()
     oprot.writeStructEnd()
 
-  def __str__(self):
+  def __str__(self): 
     return str(self.__dict__)
 
-  def __repr__(self):
+  def __repr__(self): 
     return repr(self.__dict__)
 
   def __eq__(self, other):
@@ -845,10 +845,10 @@
     oprot.writeFieldStop()
     oprot.writeStructEnd()
 
-  def __str__(self):
+  def __str__(self): 
     return str(self.__dict__)
 
-  def __repr__(self):
+  def __repr__(self): 
     return repr(self.__dict__)
 
   def __eq__(self, other):

Modified: hadoop/hive/trunk/service/src/gen-py/hive_service/ttypes.py
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/service/src/gen-py/hive_service/ttypes.py?rev=758857&r1=758856&r2=758857&view=diff
==============================================================================
--- hadoop/hive/trunk/service/src/gen-py/hive_service/ttypes.py (original)
+++ hadoop/hive/trunk/service/src/gen-py/hive_service/ttypes.py Thu Mar 26 21:12:24 2009
@@ -57,10 +57,10 @@
     oprot.writeFieldStop()
     oprot.writeStructEnd()
 
-  def __str__(self):
+  def __str__(self): 
     return str(self.__dict__)
 
-  def __repr__(self):
+  def __repr__(self): 
     return repr(self.__dict__)
 
   def __eq__(self, other):