You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@thrift.apache.org by Greg Stein <gs...@gmail.com> on 2009/01/31 09:10:31 UTC
Re: svn commit: r739520 - in /incubator/thrift/branches/py-compiler/compiler/py/src:
./ templates-plain/ templates-py/
Oh! Neat. After I land back in California, I'll go digging around and
see what I can find. Thanks for the pointer!
On Sat, Jan 31, 2009 at 09:03, David Reiss <dr...@facebook.com> wrote:
> If you're interested, I think there is already a Python-based compiler
> somewhere deep in the SVN history (pre open-source).
>
> --David
>
> gstein@apache.org wrote:
>> Author: gstein
>> Date: Sat Jan 31 07:40:26 2009
>> New Revision: 739520
>>
>> URL: http://svn.apache.org/viewvc?rev=739520&view=rev
>> Log:
>> Snapshot (back up) my work-in-progress before I hop on a plane.
>>
>> Added:
>> incubator/thrift/branches/py-compiler/compiler/py/src/gen.py
>> incubator/thrift/branches/py-compiler/compiler/py/src/parser.py
>> incubator/thrift/branches/py-compiler/compiler/py/src/scanner.py
>> incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/
>> incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py.ezt
>> incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_cvalue.ezt
>> incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_deser.ezt
>> incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_ser.ezt
>> incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/
>> incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py.ezt
>> incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_cvalue.ezt
>> incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_deser.ezt
>> incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_ser.ezt
>>
>> Added: incubator/thrift/branches/py-compiler/compiler/py/src/gen.py
>> URL: http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/gen.py?rev=739520&view=auto
>> ==============================================================================
>> --- incubator/thrift/branches/py-compiler/compiler/py/src/gen.py (added)
>> +++ incubator/thrift/branches/py-compiler/compiler/py/src/gen.py Sat Jan 31 07:40:26 2009
>> @@ -0,0 +1,169 @@
>> +#
>> +# simple generator for Thrift
>> +#
>> +
>> +import sys
>> +import os
>> +import cStringIO
>> +import operator
>> +
>> +import parser
>> +import ezt
>> +
>> +
>> +### temporary
>> +PATH = '/Users/gstein/src/asf/thrift/compiler/py/src/templates-py'
>> +t_py = ezt.Template(os.path.join(PATH, 'py.ezt'),
>> + compress_whitespace=False)
>> +t_py_ser = ezt.Template(os.path.join(PATH, 'py_ser.ezt'),
>> + compress_whitespace=False)
>> +t_py_deser = ezt.Template(os.path.join(PATH, 'py_deser.ezt'),
>> + compress_whitespace=False)
>> +t_py_cvalue = ezt.Template(os.path.join(PATH, 'py_cvalue.ezt'),
>> + compress_whitespace=False)
>> +
>> +
>> +def generate(program):
>> + t_py.generate(sys.stdout, Proxy(program))
>> +
>> +
>> +class AutoVars(object):
>> + def __init__(self):
>> + self._counter = 0
>> + self._mapping = { }
>> + self._saved = [ ]
>> +
>> + def open_context(self):
>> + self._saved.append(self._mapping)
>> + self._mapping = { }
>> +
>> + def close_context(self):
>> + self._mapping = self._saved.pop()
>> +
>> + def __getattr__(self, name):
>> + if name.startswith('__'):
>> + raise AttributeError(name)
>> +
>> + if name in self._mapping:
>> + return self._mapping[name]
>> + var = '%s%d' % (name, self._counter)
>> + self._counter += 1
>> + self._mapping[name] = var
>> + return var
>> +
>> +
>> +class Proxy(object):
>> + def __init__(self, ob):
>> + self._ob = ob
>> +
>> + for name, value in vars(ob).items():
>> + proxy = custom_proxy(value)
>> + if proxy:
>> + value = proxy(value)
>> + elif isinstance(value, list) and value:
>> + # lists are homogenous, so check the first item
>> + proxy = custom_proxy(value[0])
>> + if proxy:
>> + value = [proxy(ob) for ob in value]
>> + elif hasattr(value[0], '__dict__'):
>> + value = [Proxy(ob) for ob in value]
>> + setattr(self, name, value)
>> +
>> + def __getattr__(self, name):
>> + if name == 'auto':
>> + return g_auto
>> + raise AttributeError(name)
>> +
>> +
>> +class ProxyFieldType(Proxy):
>> + def __getattr__(self, name):
>> + if name == 'serializer':
>> + return Subtemplate(t_py_ser, self)
>> + if name == 'deserializer':
>> + return Subtemplate(t_py_deser, self)
>> + return Proxy.__getattr__(self, name)
>> +
>> +
>> +class Subtemplate(object):
>> + def __init__(self, template, data):
>> + self._template = template
>> + self._data = data
>> +
>> + def __getattr__(self, name):
>> + # jam the name of the result variable into the data params
>> + self._data.result_var = getattr(g_auto, name)
>> +
>> + # use a new variable context for this template generation
>> + g_auto.open_context()
>> + value = gen_value(self._template, self._data)
>> + g_auto.close_context()
>> +
>> + return value
>> +
>> +
>> +class ProxyField(Proxy):
>> + def __getattr__(self, name):
>> + if name == 'type_enum':
>> + return TYPE_ENUM.get(self._ob.field_type.ident,
>> + self._ob.field_type.ident.tvalue)
>> + return Proxy.__getattr__(self, name)
>> +
>> +
>> +class ProxyStruct(Proxy):
>> + def __getattr__(self, name):
>> + if name == 'sorted_fields':
>> + highest = max(int(f.field_id or -1) for f in self._ob.fields)
>> + fields = [None] * (highest + 1)
>> + for field in self._ob.fields:
>> + if field.field_id:
>> + id = int(field.field_id)
>> + if id > 0:
>> + fields[id] = ProxyField(field)
>> + return fields
>> + return Proxy.__getattr__(self, name)
>> +
>> +
>> +class ProxyConstValue(Proxy):
>> + def __getattr__(self, name):
>> + if name == 'cvalue':
>> + return gen_value(t_py_cvalue, self)
>> + return Proxy.__getattr__(self, name)
>> +
>> +
>> +def custom_proxy(value):
>> + if isinstance(value, parser.FieldType):
>> + return ProxyFieldType
>> + if isinstance(value, parser.Field):
>> + return ProxyField
>> + if isinstance(value, parser.Struct):
>> + return ProxyStruct
>> + if isinstance(value, parser.ConstValue):
>> + return ProxyConstValue
>> + return None
>> +
>> +
>> +TYPE_ENUM = {
>> + parser.ID_STRING: 'TType.STRING',
>> + parser.ID_BOOL: 'TType.BOOL',
>> + parser.ID_BYTE: 'TType.BYTE',
>> + parser.ID_I16: 'TType.I16',
>> + parser.ID_I32: 'TType.I32',
>> + parser.ID_I64: 'TType.I64',
>> + parser.ID_DOUBLE: 'TType.DOUBLE',
>> + parser.ID_MAP: 'TType.MAP',
>> + parser.ID_SET: 'TType.SET',
>> + parser.ID_LIST: 'TType.LIST',
>> + # TType.STRUCT and TType.I32 for enums
>> + }
>> +
>> +
>> +def gen_value(template, ob):
>> + buf = cStringIO.StringIO()
>> + template.generate(buf, ob)
>> + return buf.getvalue()
>> +
>> +
>> +if __name__ == '__main__':
>> + import sys
>> + program = parser.parse(open(sys.argv[1]).read())
>> + generate(program)
>>
>> Added: incubator/thrift/branches/py-compiler/compiler/py/src/parser.py
>> URL: http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/parser.py?rev=739520&view=auto
>> ==============================================================================
>> --- incubator/thrift/branches/py-compiler/compiler/py/src/parser.py (added)
>> +++ incubator/thrift/branches/py-compiler/compiler/py/src/parser.py Sat Jan 31 07:40:26 2009
>> @@ -0,0 +1,497 @@
>> +#
>> +# simple parser for Thrift.
>> +#
>> +
>> +# Note: the scanner module is designed to allow this wildcard import
>> +from scanner import *
>> +
>> +
>> +def parse(contents):
>> +
>> + scanner = Scanner(contents)
>> + program = Program()
>> +
>> + while True:
>> +
>> + t = scanner.get()
>> + if t is None:
>> + return program
>> +
>> + ### delta: we don't enforce HeaderList followed by DefinitionList
>> + ### delta: deprecated namespaces are not parsed
>> +
>> + if t == ID_INCLUDE:
>> + inc = scanner.value_of(TYPE_LIT)
>> + program.add_include(inc)
>> + elif t == ID_NAMESPACE:
>> + lang = scanner.value_of(TYPE_ID)
>> + ns = scanner.value_of(TYPE_ID)
>> + program.add_namespace(lang, ns)
>> + elif t == ID_CPP_INCLUDE:
>> + inc = scanner.value_of(TYPE_LIT)
>> + program.add_cpp_include(inc)
>> + elif t == ID_PHP_NAMESPACE:
>> + ns = scanner.value_of(TYPE_ID)
>> + program.set_php_namespace(ns)
>> + elif t == ID_XSD_NAMESPACE:
>> + ns = scanner.value_of(TYPE_LIT)
>> + program.set_xsd_namespace(ns)
>> + elif t == ID_CONST:
>> + doc = scanner.doc
>> + ft = parse_field_type(scanner, True)
>> + ident = scanner.value_of(TYPE_ID)
>> + scanner.eat_expected(SYM_EQ)
>> + value = parse_const_value(scanner)
>> + scanner.eat_commasemi()
>> + program.add_const(ident, ft, value, doc)
>> + elif t == ID_TYPEDEF:
>> + doc = scanner.doc
>> + ft = parse_field_type(scanner, False)
>> + ident = scanner.value_of(TYPE_ID)
>> + program.add_typedef(ident, ft, doc)
>> + elif t == ID_ENUM:
>> + enum_doc = scanner.doc
>> + enum_ident = scanner.value_of(TYPE_ID)
>> + scanner.eat_expected(SYM_LBRACE)
>> + values = [ ]
>> + while True:
>> + t = scanner.get(eof_allowed=False)
>> + if t == SYM_RBRACE:
>> + break
>> + if t.ttype != TYPE_ID:
>> + raise ExpectedType(TYPE_ID, t.ttype, scanner.lineno)
>> + doc = scanner.doc
>> + ident = t.tvalue
>> + t = scanner.get(eof_allowed=False)
>> + if t == SYM_EQ:
>> + value = scanner.value_of(TYPE_INT)
>> + else:
>> + scanner.pushback(t)
>> + value = None
>> + scanner.eat_commasemi()
>> + values.append(EnumValue(ident, value, doc))
>> + program.add_enum(enum_ident, values, enum_doc)
>> + elif t == ID_SENUM:
>> + doc = scanner.doc
>> + ident = scanner.value_of(TYPE_ID)
>> + scanner.eat_expected(SYM_LBRACE)
>> + values = [ ]
>> + while True:
>> + t = scanner.get(eof_allowed=False)
>> + if t == SYM_RBRACE:
>> + break
>> + if t.ttype != TYPE_LIT:
>> + raise ExpectedType(TYPE_LIT, t.ttype, scanner.lineno)
>> + scanner.eat_commasemi()
>> + values.append(t.tvalue)
>> + program.add_senum(ident, values, doc)
>> + elif t == ID_STRUCT:
>> + doc = scanner.doc
>> + ident = scanner.value_of(TYPE_ID)
>> + t = scanner.get(eof_allowed=False)
>> + if t == ID_XSD_ALL:
>> + xsd_all = True
>> + else:
>> + xsd_all = False
>> + scanner.pushback(t)
>> + fields = parse_field_list(scanner, SYM_LBRACE, SYM_RBRACE)
>> + annotations = parse_annotations(scanner)
>> + program.add_struct(ident, fields, annotations, doc)
>> + elif t == ID_EXCEPTION:
>> + doc = scanner.doc
>> + ident = scanner.value_of(TYPE_ID)
>> + fields = parse_field_list(scanner, SYM_LBRACE, SYM_RBRACE)
>> + program.add_exception(ident, fields, doc)
>> + elif t == ID_SERVICE:
>> + svc_doc = scanner.doc
>> + svc_ident = scanner.value_of(TYPE_ID)
>> + t = scanner.get(eof_allowed=False)
>> + if t == ID_EXTENDS:
>> + extends = t.tvalue
>> + t = scanner.get(eof_allowed=False)
>> + else:
>> + extends = None
>> + if t != SYM_LBRACE:
>> + raise ExpectedError(SYM_LBRACE, t, scanner.lineno)
>> + functions = [ ]
>> + while True:
>> + t = scanner.get(eof_allowed=False)
>> + doc = scanner.doc
>> + if t == SYM_RBRACE:
>> + break
>> + if t == ID_ASYNC:
>> + async = True
>> + t = scanner.get(eof_allowed=False)
>> + else:
>> + async = False
>> + if t == ID_VOID:
>> + ft = FieldType(ident=ID_VOID)
>> + else:
>> + scanner.pushback(t)
>> + ft = parse_field_type(scanner, True)
>> + ident = scanner.value_of(TYPE_ID)
>> + params = parse_field_list(scanner, SYM_LPAREN, SYM_RPAREN)
>> + t = scanner.get(eof_allowed=False)
>> + if t == ID_THROWS:
>> + throws = parse_field_list(scanner, SYM_LPAREN, SYM_RPAREN)
>> + else:
>> + throws = None
>> + scanner.pushback(t)
>> + scanner.eat_commasemi()
>> + functions.append(Function(ident, async, ft, params, throws, doc))
>> + program.add_service(svc_ident, extends, functions, svc_doc)
>> + else:
>> + raise IncorrectSyntax(scanner.lineno)
>> +
>> +
>> +def parse_field_type(scanner, ident_allowed):
>> + ident = scanner.get_type(TYPE_ID)
>> + if ident in BASE_TYPES:
>> + return FieldType(ident=ident)
>> +
>> + cpp_type = None
>> +
>> + if ident == ID_MAP:
>> + t = scanner.get(eof_allowed=False)
>> + if t == ID_CPP_TYPE:
>> + cpp_type = scanner.value_of(TYPE_LITERAL)
>> + t = scanner.get()
>> + if t != SYM_LT:
>> + raise ExpectedError(SYM_LT, t, scanner.lineno)
>> + map_from = parse_field_type(scanner, True)
>> + scanner.eat_expected(SYM_COMMA)
>> + map_to = parse_field_type(scanner, True)
>> + scanner.eat_expected(SYM_GT)
>> + return FieldType(cpp_type=cpp_type, map_from=map_from, map_to=map_to,
>> + annotations=parse_annotations(scanner))
>> +
>> + if ident == ID_SET:
>> + t = scanner.get(eof_allowed=False)
>> + if t == ID_CPP_TYPE:
>> + cpp_type = scanner.value_of(TYPE_LITERAL)
>> + t = scanner.get()
>> + if t != SYM_LT:
>> + raise ExpectedError(SYM_LT, t, scanner.lineno)
>> + set_of = parse_field_type(scanner, True)
>> + scanner.eat_expected(SYM_GT)
>> + return FieldType(cpp_type=cpp_type, set_of=set_of,
>> + annotations=parse_annotations(scanner))
>> +
>> + if ident == ID_LIST:
>> + scanner.eat_expected(SYM_LT)
>> + list_of = parse_field_type(scanner, True)
>> + scanner.eat_expected(SYM_GT)
>> + t = scanner.get()
>> + if t == ID_CPP_TYPE:
>> + cpp_type = scanner.value_of(TYPE_LITERAL)
>> + elif t is not None:
>> + scanner.pushback(t)
>> + return FieldType(cpp_type=cpp_type, list_of=list_of,
>> + annotations=parse_annotations(scanner))
>> +
>> + # random identifiers are allowed for FieldType, but not DefinitionType
>> + if ident_allowed:
>> + return FieldType(ident=ident)
>> +
>> + raise IncorrectSyntax(scanner.lineno)
>> +
>> +
>> +def parse_const_value(scanner):
>> + value = scanner.get(eof_allowed=False)
>> + if value.ttype in [TYPE_INT, TYPE_HEX, TYPE_DUB, TYPE_LIT, TYPE_ID]:
>> + return ConstValue(ConstValue.CTYPE_BASE, value)
>> +
>> + if value == SYM_LBRKT:
>> + values = [ ]
>> + while True:
>> + t = scanner.get(eof_allowed=False)
>> + if t == SYM_RBRKT:
>> + return ConstValue(ConstValue.CTYPE_LIST, values)
>> + scanner.pushback(t)
>> + scanner.eat_commasemi()
>> + values.append(parse_const_value(scanner))
>> +
>> + if value == SYM_LBRACE:
>> + values = [ ]
>> + while True:
>> + t = scanner.get(eof_allowed=False)
>> + if t == SYM_RBRACE:
>> + return ConstValue(ConstValue.CTYPE_MAP, values)
>> + scanner.pushback(t)
>> + key = parse_const_value(scanner)
>> + scanner.eat_expected(SYM_COLON)
>> + value = parse_const_value(scanner)
>> + scanner.eat_commasemi()
>> + values.append(KeyValuePair(key, value))
>> +
>> + raise IncorrectSyntax(scanner.lineno)
>> +
>> +
>> +def parse_field_list(scanner, start, end):
>> + scanner.eat_expected(start)
>> +
>> + fields = [ ]
>> + while True:
>> + t = scanner.get(eof_allowed=False)
>> + if t == end:
>> + return fields
>> + doc = scanner.doc
>> + if t.ttype == TYPE_INT:
>> + field_id = t.tvalue
>> + scanner.eat_expected(SYM_COLON)
>> + t = scanner.get(eof_allowed=False)
>> + else:
>> + field_id = None
>> + if t == ID_REQUIRED or t == ID_OPTIONAL:
>> + ### delta: we don't warn when this occurs in an arglist
>> + requiredness = t
>> + else:
>> + requiredness = None
>> + scanner.pushback(t)
>> + ft = parse_field_type(scanner, True)
>> + ident = scanner.value_of(TYPE_ID)
>> + t = scanner.get()
>> + if t == SYM_EQ:
>> + value = parse_const_value(scanner)
>> + t = scanner.get()
>> + else:
>> + value = None
>> + if t == ID_XSD_OPTIONAL:
>> + xsd_optional = True
>> + t = scanner.get()
>> + else:
>> + xsd_optional = False
>> + if t == ID_XSD_NILLABLE:
>> + xsd_nillable = True
>> + t = scanner.get()
>> + else:
>> + xsd_nillable = False
>> + if t == ID_XSD_ATTRS:
>> + xsd_attrs = parse_field_list(scanner, SYM_LBRACE, SYM_RBRACE)
>> + else:
>> + xsd_attrs = None
>> + if t is not None:
>> + scanner.pushback(t)
>> + scanner.eat_commasemi()
>> + fields.append(Field(ident, ft, doc, field_id, requiredness, value,
>> + xsd_optional, xsd_nillable, xsd_attrs))
>> +
>> +
>> +def parse_annotations(scanner):
>> + t = scanner.get()
>> + if t is None:
>> + return None
>> + if t != SYM_LPAREN:
>> + scanner.pushback(t)
>> + return None
>> + annotations = [ ]
>> + while True:
>> + ident = scanner.value_of(TYPE_ID)
>> + scanner.eat_expected(SYM_EQ)
>> + value = scanner.value_of(TYPE_LIT)
>> + annotations.append(KeyValuePair(ident, value))
>> +
>> + scanner.eat_commasemi()
>> + t = scanner.get()
>> + if t == SYM_RPAREN:
>> + return annotations
>> + scanner.pushback(t)
>> +
>> +
>> +class Program(object):
>> + def __init__(self):
>> + self.includes = [ ]
>> + self.namespaces = [ ]
>> + self.cpp_includes = [ ]
>> + self.php_namespace = None
>> + self.xsd_namespace = None
>> + self.consts = [ ]
>> + self.typedefs = [ ]
>> + self.enums = [ ]
>> + self.structs = [ ]
>> + self.exceptions = [ ]
>> + self.services = [ ]
>> +
>> + def add_include(self, include):
>> + self.includes.append(include)
>> +
>> + def add_namespace(self, lang, namespace):
>> + self.namespaces.append(Namespace(lang, namespace))
>> +
>> + def add_cpp_include(self, include):
>> + self.cpp_includes.append(include)
>> +
>> + def set_php_namespace(self, namespace):
>> + self.php_namespace = namespace
>> +
>> + def set_xsd_namespace(self, namespace):
>> + self.xsd_namespace = namespace
>> +
>> + def add_const(self, ident, field_type, value, doc):
>> + self.consts.append(ConstDef(ident, field_type, value, doc))
>> +
>> + def add_typedef(self, ident, field_type, doc):
>> + self.typedefs.append(Typedef(ident, field_type, doc))
>> +
>> + def add_enum(self, ident, value, doc):
>> + self.enums.append(Enum(ident, value, doc))
>> +
>> + def add_senum(self, ident, values, doc):
>> + self.typedefs.append(Typedef(ident, FieldType(values=values), doc))
>> +
>> + def add_struct(self, ident, fields, annotations, doc):
>> + self.structs.append(Struct(ident, fields, annotations, doc))
>> +
>> + def add_exception(self, ident, fields, doc):
>> + self.exceptions.append(Exception(ident, fields, doc))
>> +
>> + def add_service(self, ident, extends, functions, doc):
>> + self.services.append(Service(ident, extends, functions, doc))
>> +
>> +
>> +class Service(object):
>> + def __init__(self, ident, extends, functions, doc):
>> + self.ident = ident
>> + self.extends = extends
>> + self.functions = functions
>> + self.doc = doc
>> +
>> +
>> +class Function(object):
>> + def __init__(self, ident, async, field_type, params, throws, doc):
>> + self.ident = ident
>> + self.async = async
>> + self.field_type = field_type
>> + self.params = params
>> + self.throws = throws
>> + self.doc = doc
>> +
>> +
>> +class Enum(object):
>> + def __init__(self, ident, values, doc):
>> + self.ident = ident
>> + self.values = values
>> + self.doc = doc
>> +
>> + for i in range(1, len(values)):
>> + if values[i].value is None:
>> + ### keep as integer?
>> + values[i].value = str(int(values[i - 1].value) + 1)
>> +
>> +
>> +class EnumValue(object):
>> + def __init__(self, ident, value, doc):
>> + self.ident = ident
>> + self.value = value
>> + self.doc = doc
>> +
>> +
>> +class Field(object):
>> + def __init__(self, ident, field_type, doc, field_id, requiredness, value,
>> + xsd_optional, xsd_nillable, xsd_attrs):
>> + assert value is None or isinstance(value, ConstValue)
>> +
>> + self.ident = ident
>> + self.field_type = field_type
>> + self.doc = doc
>> + self.field_id = field_id
>> + self.requiredness = requiredness
>> + self.value = value
>> + self.xsd_optional = xsd_optional
>> + self.xsd_nillable = xsd_nillable
>> + self.xsd_attrs = xsd_attrs
>> +
>> +
>> +class FieldType(object):
>> + def __init__(self, ident=None, cpp_type=None, map_from=None, map_to=None,
>> + set_of=None, list_of=None, annotations=None, values=None):
>> + if map_from is not None:
>> + self.ident = ID_MAP
>> + elif set_of is not None:
>> + self.ident = ID_SET
>> + elif list_of is not None:
>> + self.ident = ID_LIST
>> + elif values is not None:
>> + self.ident = ID_STRING
>> + else:
>> + assert ident is not None
>> + self.ident = ident
>> + self.cpp_type = cpp_type
>> + self.map_from = map_from
>> + self.map_to = map_to
>> + self.set_of = set_of
>> + self.list_of = list_of
>> + self.annotations = annotations
>> + self.values = values
>> +
>> +
>> +class KeyValuePair(object):
>> + def __init__(self, key, value):
>> + self.key = key
>> + self.value = value
>> +
>> +
>> +class ConstDef(object):
>> + def __init__(self, ident, field_type, value, doc):
>> + assert isinstance(value, ConstValue)
>> +
>> + self.ident = ident
>> + self.field_type = field_type
>> + self.value = value
>> + self.doc = doc
>> +
>> +
>> +class ConstValue(object):
>> + CTYPE_BASE = 'base'
>> + CTYPE_LIST = 'list'
>> + CTYPE_MAP = 'map'
>> +
>> + def __init__(self, ctype, value):
>> + self.ctype = ctype
>> + self.value = value
>> +
>> +
>> +class Typedef(object):
>> + def __init__(self, ident, field_type, doc):
>> + self.ident = ident
>> + self.field_type = field_type
>> + self.doc = doc
>> +
>> +
>> +class Struct(object):
>> + def __init__(self, ident, fields, annotations, doc):
>> + self.ident = ident
>> + self.fields = fields
>> + self.annotations = annotations
>> + self.doc = doc
>> +
>> +
>> +class Exception(object):
>> + def __init__(self, ident, fields, doc):
>> + self.ident = ident
>> + self.fields = fields
>> + self.doc = doc
>> +
>> +
>> +class Namespace(object):
>> + def __init__(self, lang, namespace):
>> + self.lang = lang
>> + self.namespace = namespace
>> +
>> +
>> +BASE_TYPES = [
>> + ID_STRING,
>> + ID_BINARY,
>> + ID_SLIST,
>> + ID_BOOL,
>> + ID_BYTE,
>> + ID_I16,
>> + ID_I32,
>> + ID_I64,
>> + ID_DOUBLE,
>> + ]
>> +
>> +
>> +if __name__ == '__main__':
>> + import sys
>> + parse(open(sys.argv[1]).read())
>>
>> Added: incubator/thrift/branches/py-compiler/compiler/py/src/scanner.py
>> URL: http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/scanner.py?rev=739520&view=auto
>> ==============================================================================
>> --- incubator/thrift/branches/py-compiler/compiler/py/src/scanner.py (added)
>> +++ incubator/thrift/branches/py-compiler/compiler/py/src/scanner.py Sat Jan 31 07:40:26 2009
>> @@ -0,0 +1,271 @@
>> +#
>> +# simple scanner for Thrift. emits tokens.
>> +#
>> +
>> +__all__ = ['Scanner', 'SimpleScanner', 'Token', 'TYPE_INT',
>> + 'ExpectedError', 'ExpectedType', 'UnexpectedEOF',
>> + 'UnknownToken', 'IncorrectSyntax',
>> + ]
>> +
>> +import re
>> +
>> +re_int = re.compile('[+-]?[0-9]+$') # special handling
>> +re_hex = re.compile('0x[0-9A-Fa-f]+')
>> +re_dub = re.compile(r'[+-]?[0-9]*(\.[0-9]+)?([eE][+-]?[0-9]+)?')
>> +
>> +re_white = re.compile('[ \t\r\n]+')
>> +re_silly = re.compile(r'/\*+\*/')
>> +re_multi = re.compile(r'/\*[^*]/*([^*/]|[^*]/|\*[^/])*\*+/')
>> +re_comment = re.compile('//[^\n]*')
>> +re_unix = re.compile('#[^\n]*')
>> +
>> +re_doc = re.compile(r'/\*\*([^*/]|[^*]/|\*[^/])*\*+/')
>> +
>> +re_ident = re.compile('[a-zA-Z_][\.a-zA-Z_0-9]*')
>> +re_symbol = re.compile(r'[:;,{}()=<>\[\]]')
>> +re_dliteral = re.compile('"[^"]*"')
>> +re_sliteral = re.compile("'[^']*'")
>> +re_st_ident = re.compile('[a-zA-Z-][.a-zA-Z_0-9-]*')
>> +
>> +skip_re = [re_white, re_silly, re_multi, re_comment, re_unix]
>> +
>> +types = [
>> + ('HEX', re_hex), # keep before re_dub
>> + ('DUB', re_dub),
>> + ('DOC', re_doc),
>> + ('ID', re_ident),
>> + ('SYM', re_symbol),
>> + ('LIT', re_dliteral),
>> + ('LIT', re_sliteral),
>> + ('STID', re_st_ident),
>> + ]
>> +
>> +for key, pattern in types:
>> + globals()['TYPE_' + key] = key
>> + __all__.append('TYPE_' + key)
>> +TYPE_INT = 'INT'
>> +
>> +
>> +class SimpleScanner(object):
>> +
>> + def __init__(self, contents):
>> + self.contents = contents
>> + self.lineno = 1
>> +
>> + def get(self):
>> + """Get the next token.
>> +
>> + Consumes and returns the next token. Note that leading whitespace is
>> + skipped.
>> +
>> + Returns None if there are no more tokens.
>> + """
>> + self._skip()
>> +
>> + if not self.contents:
>> + return None
>> +
>> + for ttype, pattern in types:
>> + m = pattern.match(self.contents)
>> + if m:
>> + if m.end() == 0:
>> + continue
>> + tvalue = m.group()
>> + if pattern is re_dub and re_int.match(tvalue):
>> + ttype = TYPE_INT
>> + elif ttype == TYPE_LIT:
>> + # strip quotes
>> + tvalue = tvalue[1:-1]
>> + ### fold TYPE_HEX into TYPE_INT? convert INT/DUB away from string?
>> + token = Token(ttype, tvalue)
>> + self._chomp(m.end())
>> + return token
>> +
>> + raise UnknownToken(self.lineno)
>> +
>> + def _skip(self):
>> + "Skip over leading whitespace."
>> +
>> + while True:
>> + for pattern in skip_re:
>> + m = pattern.match(self.contents)
>> + if m:
>> + self._chomp(m.end())
>> + break
>> + else:
>> + # nothing matched. all done.
>> + return
>> +
>> + def _chomp(self, amt):
>> + "Chomp AMT bytes off the front of the contents. Count newlines."
>> + self.lineno += self.contents[:amt].count('\n')
>> + self.contents = self.contents[amt:]
>> +
>> +
>> +class Scanner(SimpleScanner):
>> + def __init__(self, contents):
>> + SimpleScanner.__init__(self, contents)
>> +
>> + self.doc = None
>> + self.pending = None
>> +
>> + def get(self, eof_allowed=True):
>> + if self.pending is not None:
>> + token = self.pending
>> + self.pending = None
>> + return token
>> +
>> + self.doc = None
>> + while True:
>> + t = SimpleScanner.get(self)
>> + if t is None:
>> + if eof_allowed:
>> + return None
>> + raise UnexpectedEOF(self.lineno)
>> + if t.ttype != TYPE_DOC:
>> + #print 'TOKEN:', t
>> + return t
>> + self.doc = t
>> +
>> + def get_type(self, ttype):
>> + "Get the next token, ensuring it is of the given type."
>> + t = self.get(eof_allowed=False)
>> + if t.ttype != ttype:
>> + raise ExpectedType(ttype, t.ttype, self.lineno)
>> + return t
>> +
>> + def value_of(self, ttype):
>> + "Get the next token's value, ensuring it is of the given type."
>> + return self.get_type(ttype).tvalue
>> +
>> + def pushback(self, token):
>> + "Push a token back into the scanner; it was unused."
>> + assert token is not None
>> + assert self.pending is None
>> + self.pending = token
>> +
>> + def eat_commasemi(self):
>> + "Eat a comma or a semicolon, if present."
>> + t = self.get()
>> + if t != SYM_COMMA and t != SYM_SEMI:
>> + self.pushback(t)
>> +
>> + def eat_expected(self, token):
>> + "Eat the expected token, or raise a ExpectedError."
>> + t = self.get()
>> + if t != token:
>> + raise ExpectedError(token, t, self.lineno)
>> +
>> +
>> +class Token(object):
>> + def __init__(self, ttype, tvalue=None):
>> + self.ttype = ttype
>> + self.tvalue = tvalue
>> +
>> + def __str__(self):
>> + if self.tvalue is None:
>> + return 'T(%s)' % self.ttype
>> + return 'T(%s, "%s")' % (self.ttype, self.tvalue)
>> +
>> + def __eq__(self, other):
>> + return self.ttype == other.ttype and self.tvalue == other.tvalue
>> +
>> + def __ne__(self, other):
>> + return self.ttype != other.ttype or self.tvalue != other.tvalue
>> +
>> + def __hash__(self):
>> + return hash((self.ttype, self.tvalue))
>> +
>> +
>> +for ident in ['namespace',
>> + 'cpp_namespace',
>> + 'cpp_include',
>> + 'cpp_type',
>> + 'java_package',
>> + 'cocoa_prefix',
>> + 'csharp_namespace',
>> + 'php_namespace',
>> + 'py_module',
>> + 'perl_package',
>> + 'ruby_namespace',
>> + 'smalltalk_category',
>> + 'smalltalk_prefix',
>> + 'xsd_all',
>> + 'xsd_optional',
>> + 'xsd_nillable',
>> + 'xsd_namespace',
>> + 'xsd_attrs',
>> + 'include',
>> + 'void',
>> + 'bool',
>> + 'byte',
>> + 'i16',
>> + 'i32',
>> + 'i64',
>> + 'double',
>> + 'string',
>> + 'binary',
>> + 'slist',
>> + 'senum',
>> + 'map',
>> + 'list',
>> + 'set',
>> + 'async',
>> + 'typedef',
>> + 'struct',
>> + 'exception',
>> + 'extends',
>> + 'throws',
>> + 'service',
>> + 'enum',
>> + 'const',
>> + 'required',
>> + 'optional',
>> + ]:
>> + name = 'ID_' + ident.upper()
>> + globals()[name] = Token(TYPE_ID, ident)
>> + __all__.append(name)
>> +
>> +
>> +for name, sym in [('COLON', ':'),
>> + ('SEMI', ';'),
>> + ('COMMA', ','),
>> + ('LBRACE', '{'),
>> + ('RBRACE', '}'),
>> + ('LPAREN', '('),
>> + ('RPAREN', ')'),
>> + ('LBRKT', '['),
>> + ('RBRKT', ']'),
>> + ('EQ', '='),
>> + ('LT', '<'),
>> + ('GT', '>'),
>> + ]:
>> + globals()['SYM_' + name] = Token(TYPE_SYM, sym)
>> + __all__.append('SYM_' + name)
>> +
>> +
>> +class ExpectedError(Exception):
>> + "Expected token was not present."
>> +
>> +class ExpectedType(Exception):
>> + "Expected token type was not present."
>> +
>> +class UnexpectedEOF(Exception):
>> + "EOF reached unexpectedly."
>> +
>> +class UnknownToken(Exception):
>> + "Unknown token encountered."
>> +
>> +class IncorrectSyntax(Exception):
>> + "Incorrect syntax encountered."
>> +
>> +
>> +if __name__ == '__main__':
>> + import sys
>> +
>> + s = Scanner(open(sys.argv[1]).read())
>> + while True:
>> + token = s.get()
>> + if token is None:
>> + break
>> + print token
>>
>> Added: incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py.ezt
>> URL: http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py.ezt?rev=739520&view=auto
>> ==============================================================================
>> --- incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py.ezt (added)
>> +++ incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py.ezt Sat Jan 31 07:40:26 2009
>> @@ -0,0 +1,33 @@
>> +[if-any includes]Includes:[for includes]
>> + [includes][end]
>> +
>> +[end][if-any namespaces]Namespaces:[for namespaces]
>> + [namespaces.lang] [namespaces.namespace][end]
>> +
>> +[end][if-any cpp_includes]C++ Includes:[for cpp_includes]
>> + [cpp_includes][end]
>> +
>> +[end][if-any php_namespace]PHP Namespace: [php_namespace]
>> +[end][if-any xsd_namespace]XSD Namespace: [xsd_namespace]
>> +[end][if-any consts]Constants:[for consts]
>> + [consts.ident] [consts.field_type.serializer] = [consts.value.cvalue][end]
>> +
>> +[end][if-any typedefs]Typedefs:[for typedefs]
>> + [typedefs.ident] => [typedefs.field_type.serializer][end]
>> +
>> +[end][if-any enums]Enums:[for enums]
>> + [enums.ident] {[for enums.values]
>> + [enums.values.ident] = [enums.values.value],[end]
>> + }[end]
>> +
>> +[end][if-any structs]Structs:[for structs]
>> + [structs.ident] {[for structs.fields]
>> + [structs.fields.field_id]: [structs.fields.field_type.serializer] [structs.fields.ident],[end]
>> + }[end]
>> +
>> +[end][if-any exceptions]Exceptions:[for exceptions]
>> + [exceptions.ident][end]
>> +
>> +[end][if-any services]Services:[for services]
>> + [services.ident][end]
>> +[end]
>>
>> Added: incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_cvalue.ezt
>> URL: http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_cvalue.ezt?rev=739520&view=auto
>> ==============================================================================
>> --- incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_cvalue.ezt (added)
>> +++ incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_cvalue.ezt Sat Jan 31 07:40:26 2009
>> @@ -0,0 +1,3 @@
>> +[is ctype "map"]{[for value][value.key.cvalue]: [value.value.cvalue],[end]}[#
>> +#][else][is ctype "list"][[][for value][value.cvalue],[end]][#
>> +#][else][value.tvalue][end][end]
>> \ No newline at end of file
>>
>> Added: incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_deser.ezt
>> URL: http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_deser.ezt?rev=739520&view=auto
>> ==============================================================================
>> --- incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_deser.ezt (added)
>> +++ incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_deser.ezt Sat Jan 31 07:40:26 2009
>> @@ -0,0 +1 @@
>> +deserializer
>>
>> Added: incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_ser.ezt
>> URL: http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_ser.ezt?rev=739520&view=auto
>> ==============================================================================
>> --- incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_ser.ezt (added)
>> +++ incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_ser.ezt Sat Jan 31 07:40:26 2009
>> @@ -0,0 +1,6 @@
>> +[is ident.tvalue "map"]map<[map_from.serializer],[map_to.serializer]>[#
>> +][else][is ident.tvalue "set"]set<[set_of.serializer]>[#
>> +][else][is ident.tvalue "list"]list<[list_of.serializer]>[#
>> +][else][if-any values]string {[for values][values],[end]}[#
>> +][else][ident.tvalue][#
>> +][end][end][end][end]
>> \ No newline at end of file
>>
>> Added: incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py.ezt
>> URL: http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py.ezt?rev=739520&view=auto
>> ==============================================================================
>> --- incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py.ezt (added)
>> +++ incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py.ezt Sat Jan 31 07:40:26 2009
>> @@ -0,0 +1,100 @@
>> +#
>> +# Autogenerated by Thrift
>> +#
>> +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
>> +#
>> +
>> +from thrift.Thrift import *
>> +[# ### probably need some massaging to really locate this module
>> +][for includes]import [includes].ttypes
>> +[end]
>> +from thrift.transport import TTransport
>> +from thrift.protocol import TBinaryProtocol
>> +try:
>> + from thrift.protocol import fastbinary
>> +except:
>> + fastbinary = None
>> +
>> +
>> +[# ### need gen_newstyle flag
>> +][for enums]class [enums.ident](object):[for enums.values]
>> + [enums.values.ident] = [enums.values.value][end]
>> +[end]
>> +
>> +[for consts][consts.ident] = [consts.value.cvalue]
>> +[end]
>> +
>> +[# ### need gen_newstyle flag
>> +][for structs]class [structs.ident](object):
>> +
>> + thrift_spec = ([# ### should sort fields. need None markers for gaps.
>> +][if-any structs.fields][for structs.sorted_fields]
>> + [if-any structs.sorted_fields]([structs.sorted_fields.field_id], [structs.sorted_fields.type_enum], [#
>> +]'[structs.sorted_fields.ident]', [#
>> +]None, [# ### should have spec_args here
>> +][if-any structs.sorted_fields.value][structs.sorted_fields.value.cvalue][else]None[end], [#
>> +]),[else]None,[end] # ### list-index[# structs.sorted_fields.list-index][end]
>> + )
>> +[else] thrift_spec = None
>> +[end]
>> +[if-any structs.fields] def __init__(self,[#
>> +][for structs.fields] [structs.fields.ident]=[#
>> +][if-any structs.fields.value]thrift_spec[[][structs.fields.field_id]][[]4][#
>> +][else]None[end],[end]):
>> +[for structs.fields][if-any ""][# ### complex test here
>> +] if [structs.fields.ident] is self.thrift_spec[[]structs.fields.field_id][[]4]:
>> + [structs.fields.ident] = [structs.fields.value.cvalue]
>> +[end] self.[structs.fields.ident] = [structs.fields.ident]
>> +[end]
>> +[end][# close: if-any structs.fields]
>> +
>> + def read(self, iprot):
>> + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated [#
>> +]and isinstance(iprot.trans, TTransport.CReadableTransport) [#
>> +]and self.thrift_spec is not None and fastbinary is not None:
>> + fastbinary.decode_binary(self, iprot.trans, [#
>> +](self.__class__, self.thrift_spec))
>> + return
>> + iprot.readStructBegin()
>> + while True:
>> + (fname, ftype, fid) = iprot.readFieldBegin()
>> + if ftype == TType.STOP:
>> + break[for structs.fields]
>> + [if-index structs.fields first]if[else]elif[end] fid == [#
>> +][structs.fields.field_id]:
>> + if ftype == [structs.fields.type_enum]:
>> + pass # deserialize
>> + else:
>> + iprot.skip(ftype)[end]
>> + else:
>> + iprot.skip(ftype)
>> + iprot.readFieldEnd()
>> + iprot.readStructEnd()
>> +
>> + def write(self, oprot):
>> + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated [#
>> +]and self.thrift_spec is not None and fastbinary is not None:
>> + oprot.trans.write(fastbinary.encode_binary(self, [#
>> +](self.__class__, self.thrift_spec)))
>> + return
>> + oprot.writeStructBegin('[structs.ident]')[for structs.fields]
>> + if self.[structs.fields.ident] != None:
>> + oprot.writeFieldBegin('[structs.fields.ident]', [#
>> +][structs.fields.type_enum], [structs.fields.field_id])
>> + # serialize
>> + oprot.writeFieldEnd()[end]
>> + oprot.writeFieldStop()
>> + oprot.writeStructEnd()
>> +
>> + def __repr__(self):
>> + L = [[]'%s=%r' % (key, value)
>> + for key, value in self.__dict__.iteritems()]
>> + return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
>> +
>> + def __eq__(self, other):
>> + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
>> +
>> + def __ne__(self, other):
>> + return not (self == other)
>> +
>> +[end][# for structs]
>>
>> Added: incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_cvalue.ezt
>> URL: http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_cvalue.ezt?rev=739520&view=auto
>> ==============================================================================
>> --- incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_cvalue.ezt (added)
>> +++ incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_cvalue.ezt Sat Jan 31 07:40:26 2009
>> @@ -0,0 +1,10 @@
>> +[is ctype "map"]{
>> + [for value][value.key.cvalue]: [value.value.cvalue],
>> +[end]}[#
>> +#][else][is ctype "set"]set([[]
>> + [for value][value.cvalue],
>> +[end]])[#
>> +#][else][is ctype "list"][[]
>> + [for value][value.cvalue],
>> +[end]][#
>> +#][else][value.tvalue][end][end][end]
>> \ No newline at end of file
>>
>> Added: incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_deser.ezt
>> URL: http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_deser.ezt?rev=739520&view=auto
>> ==============================================================================
>> (empty)
>>
>> Added: incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_ser.ezt
>> URL: http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_ser.ezt?rev=739520&view=auto
>> ==============================================================================
>> (empty)
>>
>>
>