You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@thrift.apache.org by "ASF GitHub Bot (JIRA)" <ji...@apache.org> on 2018/04/06 13:07:00 UTC
[jira] [Commented] (THRIFT-3770) Support Python 3.4+ asyncio
support
[ https://issues.apache.org/jira/browse/THRIFT-3770?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16428279#comment-16428279 ]
ASF GitHub Bot commented on THRIFT-3770:
----------------------------------------
jeking3 closed pull request #972: THRIFT-3770 Implement Python 3.4+ asyncio support
URL: https://github.com/apache/thrift/pull/972
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/.travis.yml b/.travis.yml
index 33a32d8d88..7c2ec0ca6e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -169,7 +169,7 @@ matrix:
- env: TEST_NAME="cppcheck, flake8, TODO FIXME HACK, LoC and system info"
install:
- travis_retry sudo apt-get update
- - travis_retry sudo apt-get install -ym cppcheck sloccount python-flake8
+ - travis_retry sudo apt-get install -ym cppcheck sloccount python3-flake8
script:
# Compiler cppcheck (All)
- cppcheck --force --quiet --inline-suppr --enable=all -j2 compiler/cpp/src
@@ -182,9 +182,11 @@ matrix:
- cppcheck --force --quiet --inline-suppr --error-exitcode=1 -j2 lib/cpp/src lib/cpp/test test/cpp tutorial/cpp
- cppcheck --force --quiet --inline-suppr --error-exitcode=1 -j2 lib/c_glib/src lib/c_glib/test test/c_glib/src tutorial/c_glib
# Python code style
- - flake8 --ignore=E501 lib/py
- - flake8 tutorial/py
- - flake8 --ignore=E501 test/py
+ - shopt -s expand_aliases
+ - alias flake83="PYTHONPATH=/usr/lib/python3/dist-packages python3 -m flake8.run"
+ - flake83 --ignore=E501 lib/py
+ - flake83 tutorial/py
+ - flake83 --ignore=E501 test/py
- flake8 test/py.twisted
- flake8 test/py.tornado
- flake8 --ignore=E501 test/test.py
diff --git a/compiler/cpp/src/generate/t_py_generator.cc b/compiler/cpp/src/generate/t_py_generator.cc
index 1ee0fcb627..94863850a4 100644
--- a/compiler/cpp/src/generate/t_py_generator.cc
+++ b/compiler/cpp/src/generate/t_py_generator.cc
@@ -59,6 +59,7 @@ class t_py_generator : public t_generator {
gen_slots_ = false;
gen_tornado_ = false;
gen_twisted_ = false;
+ gen_asyncio_ = false;
gen_dynamic_ = false;
coding_ = "";
gen_dynbaseclass_ = "";
@@ -109,15 +110,17 @@ class t_py_generator : public t_generator {
gen_twisted_ = true;
} else if( iter->first.compare("tornado") == 0) {
gen_tornado_ = true;
+ } else if( iter->first.compare("asyncio") == 0) {
+ gen_asyncio_ = true;
} else if( iter->first.compare("coding") == 0) {
coding_ = iter->second;
} else {
- throw "unknown option py:" + iter->first;
+ throw "unknown option py:" + iter->first;
}
}
- if (gen_twisted_ && gen_tornado_) {
- throw "at most one of 'twisted' and 'tornado' are allowed";
+ if (gen_twisted_ + gen_tornado_ + gen_asyncio_ > 1) {
+ throw "at most one of 'twisted', 'tornado', 'asyncio' are allowed";
}
copy_options_ = option_string;
@@ -126,6 +129,8 @@ class t_py_generator : public t_generator {
out_dir_base_ = "gen-py.twisted";
} else if (gen_tornado_) {
out_dir_base_ = "gen-py.tornado";
+ } else if (gen_asyncio_) {
+ out_dir_base_ = "gen-py.asyncio";
} else {
out_dir_base_ = "gen-py";
}
@@ -298,6 +303,11 @@ class t_py_generator : public t_generator {
*/
bool gen_tornado_;
+ /**
+ * True if we should generate code for use with asyncio
+ */
+ bool gen_asyncio_;
+
/**
* True if strings should be encoded using utf-8.
*/
@@ -416,9 +426,14 @@ string t_py_generator::py_autogen_comment() {
*/
string t_py_generator::py_imports() {
ostringstream ss;
- ss << "from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, "
- "TApplicationException"
- << endl
+ if (gen_asyncio_) {
+ ss << "import asyncio" << endl;
+ }
+ ss << "from thrift.Thrift import TType, TMessageType, TFrozenDict, TException";
+ if (!gen_asyncio_) {
+ ss << ", TApplicationException";
+ }
+ ss << endl
<< "from thrift.protocol.TProtocol import TProtocolException";
if (gen_utf8strings_) {
ss << endl << "import sys";
@@ -874,6 +889,9 @@ void t_py_generator::generate_py_struct_reader(ofstream& out, t_struct* tstruct)
if (is_immutable(tstruct)) {
out << indent() << "@classmethod" << endl << indent() << "def read(cls, iprot):" << endl;
} else {
+ if (gen_asyncio_) {
+ indent(out) << "@asyncio.coroutine" << endl;
+ }
indent(out) << "def read(self, iprot):" << endl;
}
indent_up();
@@ -894,14 +912,23 @@ void t_py_generator::generate_py_struct_reader(ofstream& out, t_struct* tstruct)
}
indent_down();
- indent(out) << "iprot.readStructBegin()" << endl;
+ if (gen_asyncio_) {
+ indent(out) << "yield from ";
+ } else {
+ out << indent();
+ }
+ out << "iprot.readStructBegin()" << endl;
// Loop over reading in fields
indent(out) << "while True:" << endl;
indent_up();
// Read beginning field marker
- indent(out) << "(fname, ftype, fid) = iprot.readFieldBegin()" << endl;
+ indent(out) << "(fname, ftype, fid) = ";
+ if (gen_asyncio_) {
+ out << "yield from ";
+ }
+ out << "iprot.readFieldBegin()" << endl;
// Check for field STOP marker and break
indent(out) << "if ftype == TType.STOP:" << endl;
@@ -930,19 +957,37 @@ void t_py_generator::generate_py_struct_reader(ofstream& out, t_struct* tstruct)
generate_deserialize_field(out, *f_iter, "self.");
}
indent_down();
- out << indent() << "else:" << endl << indent() << indent_str() << "iprot.skip(ftype)" << endl;
+ out << indent() << "else:" << endl << indent() << indent_str();
+ if (gen_asyncio_) {
+ out << "yield from ";
+ }
+ out << "iprot.skip(ftype)" << endl;
indent_down();
}
// In the default case we skip the field
- out << indent() << "else:" << endl << indent() << indent_str() << "iprot.skip(ftype)" << endl;
+ out << indent() << "else:" << endl << indent() << indent_str();
+ if (gen_asyncio_) {
+ out << "yield from ";
+ }
+ out << "iprot.skip(ftype)" << endl;
// Read field end marker
- indent(out) << "iprot.readFieldEnd()" << endl;
+ if (gen_asyncio_) {
+ indent(out) << "yield from ";
+ } else {
+ out << indent();
+ }
+ out << "iprot.readFieldEnd()" << endl;
indent_down();
- indent(out) << "iprot.readStructEnd()" << endl;
+ if (gen_asyncio_) {
+ indent(out) << "yield from ";
+ } else {
+ out << indent();
+ }
+ out << "iprot.readStructEnd()" << endl;
if (is_immutable(tstruct)) {
indent(out) << "return cls(" << endl;
@@ -1057,6 +1102,9 @@ void t_py_generator::generate_service(t_service* tservice) {
} else if (gen_tornado_) {
f_service_ << "from tornado import gen" << endl;
f_service_ << "from tornado import concurrent" << endl;
+ } else if (gen_asyncio_) {
+ f_service_ << "from thrift.TAsyncio import TAsyncioApplicationException"
+ << endl;
}
// Generate the three main parts of the service
@@ -1125,7 +1173,7 @@ void t_py_generator::generate_service_interface(t_service* tservice) {
} else {
if (gen_twisted_) {
extends_if = "(Interface)";
- } else if (gen_newstyle_ || gen_dynamic_ || gen_tornado_) {
+ } else if (gen_newstyle_ || gen_dynamic_ || gen_tornado_ || gen_asyncio_) {
extends_if = "(object)";
}
}
@@ -1277,6 +1325,9 @@ void t_py_generator::generate_service_client(t_service* tservice) {
f_service_ << endl;
// Open function
+ if (gen_asyncio_) {
+ indent(f_service_) << "@asyncio.coroutine" << endl;
+ }
indent(f_service_) << "def " << function_signature(*f_iter, false) << ":" << endl;
indent_up();
generate_python_docstring(f_service_, (*f_iter));
@@ -1292,6 +1343,8 @@ void t_py_generator::generate_service_client(t_service* tservice) {
}
indent(f_service_) << "self.send_" << funname << "(";
+ } else if (gen_asyncio_) {
+ indent(f_service_) << "yield from self.send_" << funname << "(";
} else {
indent(f_service_) << "self.send_" << funname << "(";
}
@@ -1323,7 +1376,14 @@ void t_py_generator::generate_service_client(t_service* tservice) {
if (!(*f_iter)->get_returntype()->is_void()) {
f_service_ << "return ";
}
- f_service_ << "self.recv_" << funname << "()" << endl;
+ if (gen_asyncio_) {
+ f_service_ << "(yield from ";
+ }
+ f_service_ << "self.recv_" << funname << "()";
+ if (gen_asyncio_) {
+ f_service_ << ")";
+ }
+ f_service_ << endl;
}
}
indent_down();
@@ -1364,6 +1424,9 @@ void t_py_generator::generate_service_client(t_service* tservice) {
}
f_service_ << endl;
+ if (gen_asyncio_) {
+ indent(f_service_) << "@asyncio.coroutine" << endl;
+ }
indent(f_service_) << "def send_" << function_signature(*f_iter, false) << ":" << endl;
indent_up();
@@ -1393,8 +1456,11 @@ void t_py_generator::generate_service_client(t_service* tservice) {
<< endl << indent() << "oprot.trans.flush()" << endl;
} else {
f_service_ << indent() << "args.write(self._oprot)" << endl << indent()
- << "self._oprot.writeMessageEnd()" << endl << indent()
- << "self._oprot.trans.flush()" << endl;
+ << "self._oprot.writeMessageEnd()" << endl << indent();
+ if (gen_asyncio_) {
+ f_service_ << "yield from ";
+ }
+ f_service_ << "self._oprot.trans.flush()" << endl;
}
indent_down();
@@ -1411,6 +1477,9 @@ void t_py_generator::generate_service_client(t_service* tservice) {
t_function recv_function((*f_iter)->get_returntype(),
string("recv_") + (*f_iter)->get_name(),
&noargs);
+ if (gen_asyncio_) {
+ f_service_ << indent() << "@asyncio.coroutine" << endl;
+ }
f_service_ << indent() << "def " << function_signature(&recv_function) << ":" << endl;
}
indent_up();
@@ -1422,22 +1491,45 @@ void t_py_generator::generate_service_client(t_service* tservice) {
} else if (gen_tornado_) {
} else {
f_service_ << indent() << "iprot = self._iprot" << endl << indent()
- << "(fname, mtype, rseqid) = iprot.readMessageBegin()" << endl;
+ << "(fname, mtype, rseqid) = ";
+ if (gen_asyncio_) {
+ f_service_ << "yield from ";
+ }
+ f_service_ << "iprot.readMessageBegin()" << endl;
}
f_service_ << indent() << "if mtype == TMessageType.EXCEPTION:" << endl
- << indent() << indent_str() << "x = TApplicationException()" << endl;
+ << indent() << indent_str() << "x = T";
+ if (gen_asyncio_) {
+ f_service_ << "Asyncio";
+ }
+ f_service_ << "ApplicationException()" << endl;
if (gen_twisted_) {
f_service_ << indent() << indent_str() << "x.read(iprot)" << endl << indent()
- << indent_str() << "iprot.readMessageEnd()" << endl << indent() << indent_str() << "return d.errback(x)"
- << endl << indent() << "result = " << resultname << "()" << endl << indent()
+ << indent_str() << "iprot.readMessageEnd()" << endl << indent()
+ << indent_str() << "return d.errback(x)" << endl << indent()
+ << "result = " << resultname << "()" << endl << indent()
<< "result.read(iprot)" << endl << indent() << "iprot.readMessageEnd()" << endl;
} else {
- f_service_ << indent() << indent_str() << "x.read(iprot)" << endl << indent()
- << indent_str() << "iprot.readMessageEnd()" << endl << indent() << indent_str() << "raise x" << endl
- << indent() << "result = " << resultname << "()" << endl << indent()
- << "result.read(iprot)" << endl << indent() << "iprot.readMessageEnd()" << endl;
+ f_service_ << indent() << indent_str();
+ if (gen_asyncio_) {
+ f_service_ << "yield from ";
+ }
+ f_service_ << "x.read(iprot)" << endl << indent() << indent_str();
+ if (gen_asyncio_) {
+ f_service_ << "yield from ";
+ }
+ f_service_ << "iprot.readMessageEnd()" << endl << indent() << indent_str() << "raise x" << endl
+ << indent() << "result = " << resultname << "()" << endl << indent();
+ if (gen_asyncio_) {
+ f_service_ << "yield from ";
+ }
+ f_service_ << "result.read(iprot)" << endl << indent();
+ if (gen_asyncio_) {
+ f_service_ << "yield from ";
+ }
+ f_service_ << "iprot.readMessageEnd()" << endl;
}
// Careful, only return _result if not a void function
@@ -1478,7 +1570,11 @@ void t_py_generator::generate_service_client(t_service* tservice) {
<< (*f_iter)->get_name() << " failed: unknown result\"))" << endl;
} else {
f_service_ << indent()
- << "raise TApplicationException(TApplicationException.MISSING_RESULT, \""
+ << "raise T";
+ if (gen_asyncio_) {
+ f_service_ << "Asyncio";
+ }
+ f_service_ << "ApplicationException(TApplicationException.MISSING_RESULT, \""
<< (*f_iter)->get_name() << " failed: unknown result\")" << endl;
}
}
@@ -1740,16 +1836,29 @@ void t_py_generator::generate_service_server(t_service* tservice) {
// HOT: dictionary function lookup
f_service_ << indent() << "if name not in self._processMap:" << endl;
indent_up();
- f_service_ << indent() << "iprot.skip(TType.STRUCT)" << endl
- << indent() << "iprot.readMessageEnd()" << endl
- << indent()
- << "x = TApplicationException(TApplicationException.UNKNOWN_METHOD, 'Unknown "
- "function %s' % (name))"
- << endl
+ f_service_ << indent();
+ if (gen_asyncio_) {
+ f_service_ << "yield from ";
+ }
+ f_service_ << "iprot.skip(TType.STRUCT)" << endl << indent();
+ if (gen_asyncio_) {
+ f_service_ << "yield from ";
+ }
+ f_service_ << "iprot.readMessageEnd()" << endl << indent()
+ << "x = T";
+ if (gen_asyncio_) {
+ f_service_ << "Asyncio";
+ }
+ f_service_ << "ApplicationException(TApplicationException.UNKNOWN_METHOD, 'Unknown "
+ "function %s' % (name))" << endl
<< indent() << "oprot.writeMessageBegin(name, TMessageType.EXCEPTION, seqid)" << endl
<< indent() << "x.write(oprot)" << endl
<< indent() << "oprot.writeMessageEnd()" << endl
- << indent() << "oprot.trans.flush()" << endl;
+ << indent();
+ if (gen_asyncio_) {
+ f_service_ << "yield from ";
+ }
+ f_service_ << "oprot.trans.flush()" << endl;
if (gen_twisted_) {
f_service_ << indent() << "return defer.succeed(None)" << endl;
@@ -1794,6 +1903,9 @@ void t_py_generator::generate_process_function(t_service* tservice, t_function*
f_service_ << indent() << "@gen.coroutine" << endl << indent() << "def process_"
<< tfunction->get_name() << "(self, seqid, iprot, oprot):" << endl;
} else {
+ if (gen_asyncio_) {
+ f_service_ << indent() << "@asyncio.coroutine" << endl;
+ }
f_service_ << indent() << "def process_" << tfunction->get_name()
<< "(self, seqid, iprot, oprot):" << endl;
}
@@ -1803,8 +1915,15 @@ void t_py_generator::generate_process_function(t_service* tservice, t_function*
string argsname = tfunction->get_name() + "_args";
string resultname = tfunction->get_name() + "_result";
- f_service_ << indent() << "args = " << argsname << "()" << endl << indent() << "args.read(iprot)"
- << endl << indent() << "iprot.readMessageEnd()" << endl;
+ f_service_ << indent() << "args = " << argsname << "()" << endl << indent();
+ if (gen_asyncio_) {
+ f_service_ << "yield from ";
+ }
+ f_service_ << "args.read(iprot)" << endl << indent();
+ if (gen_asyncio_) {
+ f_service_ << "yield from ";
+ }
+ f_service_ << "iprot.readMessageEnd()" << endl;
t_struct* xs = tfunction->get_xceptions();
const std::vector<t_field*>& xceptions = xs->get_members();
@@ -1863,8 +1982,11 @@ void t_py_generator::generate_process_function(t_service* tservice, t_function*
f_service_ << indent() << "result.success = success" << endl << indent()
<< "oprot.writeMessageBegin(\"" << tfunction->get_name()
<< "\", TMessageType.REPLY, seqid)" << endl << indent() << "result.write(oprot)"
- << endl << indent() << "oprot.writeMessageEnd()" << endl << indent()
- << "oprot.trans.flush()" << endl;
+ << endl << indent() << "oprot.writeMessageEnd()" << endl << indent();
+ if (gen_asyncio_) {
+ f_service_ << "yield from ";
+ }
+ f_service_ << "oprot.trans.flush()" << endl;
indent_down();
// Try block for a function with exceptions
@@ -1891,8 +2013,11 @@ void t_py_generator::generate_process_function(t_service* tservice, t_function*
}
f_service_ << indent() << "oprot.writeMessageBegin(\"" << tfunction->get_name()
<< "\", TMessageType.REPLY, seqid)" << endl << indent() << "result.write(oprot)"
- << endl << indent() << "oprot.writeMessageEnd()" << endl << indent()
- << "oprot.trans.flush()" << endl;
+ << endl << indent() << "oprot.writeMessageEnd()" << endl << indent();
+ if (gen_asyncio_) {
+ f_service_ << "yield from ";
+ }
+ f_service_ << "oprot.trans.flush()" << endl;
indent_down();
}
@@ -1943,8 +2068,11 @@ void t_py_generator::generate_process_function(t_service* tservice, t_function*
if (!tfunction->is_oneway()) {
f_service_ << indent() << "oprot.writeMessageBegin(\"" << tfunction->get_name()
<< "\", TMessageType.REPLY, seqid)" << endl << indent() << "result.write(oprot)"
- << endl << indent() << "oprot.writeMessageEnd()" << endl << indent()
- << "oprot.trans.flush()" << endl;
+ << endl << indent() << "oprot.writeMessageEnd()" << endl << indent();
+ if (gen_asyncio_) {
+ f_service_ << "yield from ";
+ }
+ f_service_ << "oprot.trans.flush()" << endl;
}
// Close function
@@ -2004,13 +2132,21 @@ void t_py_generator::generate_process_function(t_service* tservice, t_function*
<< indent() << indent_str() << "msg_type = TMessageType.EXCEPTION" << endl
<< indent() << indent_str() << "logging.exception(ex)" << endl
<< indent()
- << indent_str() << "result = TApplicationException(TApplicationException.INTERNAL_ERROR, "
+ << indent_str() << "result = T";
+ if (gen_asyncio_) {
+ f_service_ << "Async";
+ }
+ f_service_ << "ApplicationException(TApplicationException.INTERNAL_ERROR, "
"'Internal error')" << endl
<< indent() << "oprot.writeMessageBegin(\"" << tfunction->get_name()
<< "\", msg_type, seqid)" << endl
<< indent() << "result.write(oprot)" << endl
<< indent() << "oprot.writeMessageEnd()" << endl
- << indent() << "oprot.trans.flush()" << endl;
+ << indent();
+ if (gen_asyncio_) {
+ f_service_ << "yield from ";
+ }
+ f_service_ << "oprot.trans.flush()" << endl;
} else {
f_service_ << indent() << "except:" << endl
<< indent() << indent_str() << "pass" << endl;
@@ -2040,7 +2176,11 @@ void t_py_generator::generate_deserialize_field(ofstream& out,
} else if (type->is_container()) {
generate_deserialize_container(out, type, name);
} else if (type->is_base_type() || type->is_enum()) {
- indent(out) << name << " = iprot.";
+ indent(out) << name << " = ";
+ if (gen_asyncio_) {
+ out << "yield from ";
+ }
+ out << "iprot.";
if (type->is_base_type()) {
t_base_type::t_base tbase = ((t_base_type*)type)->get_base();
@@ -2094,10 +2234,18 @@ void t_py_generator::generate_deserialize_field(ofstream& out,
*/
void t_py_generator::generate_deserialize_struct(ofstream& out, t_struct* tstruct, string prefix) {
if (is_immutable(tstruct)) {
- out << indent() << prefix << " = " << type_name(tstruct) << ".read(iprot)" << endl;
+ out << indent() << prefix << " = ";
+ if (gen_asyncio_) {
+ out << "yield from ";
+ }
+ out << type_name(tstruct) << ".read(iprot)" << endl;
} else {
out << indent() << prefix << " = " << type_name(tstruct) << "()" << endl
- << indent() << prefix << ".read(iprot)" << endl;
+ << indent();
+ if (gen_asyncio_) {
+ out << "yield from ";
+ }
+ out << prefix << ".read(iprot)" << endl;
}
}
@@ -2119,13 +2267,25 @@ void t_py_generator::generate_deserialize_container(ofstream& out, t_type* ttype
// Declare variables, read header
if (ttype->is_map()) {
out << indent() << prefix << " = {}" << endl << indent() << "(" << ktype << ", " << vtype
- << ", " << size << ") = iprot.readMapBegin()" << endl;
+ << ", " << size << ") = ";
+ if (gen_asyncio_) {
+ out << "yield from ";
+ }
+ out << "iprot.readMapBegin()" << endl;
} else if (ttype->is_set()) {
out << indent() << prefix << " = set()" << endl << indent() << "(" << etype << ", " << size
- << ") = iprot.readSetBegin()" << endl;
+ << ") = ";
+ if (gen_asyncio_) {
+ out << "yield from ";
+ }
+ out << "iprot.readSetBegin()" << endl;
} else if (ttype->is_list()) {
out << indent() << prefix << " = []" << endl << indent() << "(" << etype << ", " << size
- << ") = iprot.readListBegin()" << endl;
+ << ") = ";
+ if (gen_asyncio_) {
+ out << "yield from ";
+ }
+ out << "iprot.readListBegin()" << endl;
}
// For loop iterates over elements
@@ -2147,12 +2307,22 @@ void t_py_generator::generate_deserialize_container(ofstream& out, t_type* ttype
// Read container end
if (ttype->is_map()) {
- indent(out) << "iprot.readMapEnd()" << endl;
+ if (gen_asyncio_) {
+ indent(out) << "yield from ";
+ } else {
+ out << indent();
+ }
+ out << "iprot.readMapEnd()" << endl;
if (is_immutable(ttype)) {
indent(out) << prefix << " = TFrozenDict(" << prefix << ")" << endl;
}
} else if (ttype->is_set()) {
- indent(out) << "iprot.readSetEnd()" << endl;
+ if (gen_asyncio_) {
+ indent(out) << "yield from ";
+ } else {
+ out << indent();
+ }
+ out << "iprot.readSetEnd()" << endl;
if (is_immutable(ttype)) {
indent(out) << prefix << " = frozenset(" << prefix << ")" << endl;
}
@@ -2160,7 +2330,12 @@ void t_py_generator::generate_deserialize_container(ofstream& out, t_type* ttype
if (is_immutable(ttype)) {
indent(out) << prefix << " = tuple(" << prefix << ")" << endl;
}
- indent(out) << "iprot.readListEnd()" << endl;
+ if (gen_asyncio_) {
+ indent(out) << "yield from ";
+ } else {
+ out << indent();
+ }
+ out << "iprot.readListEnd()" << endl;
}
}
@@ -2611,6 +2786,7 @@ THRIFT_REGISTER_GENERATOR(
"Python",
" twisted: Generate Twisted-friendly RPC services.\n"
" tornado: Generate code for use with Tornado.\n"
+ " asyncio: Generate code for use with asyncio.\n"
" no_utf8strings: Do not Encode/decode strings using utf8 in the generated code. Basically no effect for Python 3.\n"
" coding=CODING: Add file encoding declare in generated file.\n"
" slots: Generate code using slots for instance members.\n"
diff --git a/configure.ac b/configure.ac
index bc52adf6ef..f363226bfa 100755
--- a/configure.ac
+++ b/configure.ac
@@ -858,6 +858,7 @@ if test "$have_python" = "yes" ; then
echo "Python Library:"
echo " Using Python .............. : $PYTHON"
echo " Using Trial ............... : $TRIAL"
+ echo " Python3 ................... : $have_py3"
fi
if test "$have_php" = "yes" ; then
echo
diff --git a/lib/py/src/TAsyncio.py b/lib/py/src/TAsyncio.py
new file mode 100644
index 0000000000..fd5d05b281
--- /dev/null
+++ b/lib/py/src/TAsyncio.py
@@ -0,0 +1,561 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import asyncio
+from io import BytesIO
+import logging
+from struct import pack, unpack
+
+from thrift.Thrift import TType, TApplicationException
+from thrift.protocol.TBinaryProtocol import TBinaryProtocol, \
+ TBinaryProtocolFactory
+from thrift.protocol.TCompactProtocol import TCompactProtocol, \
+ TCompactProtocolFactory, fromZigZag, reader, CLEAR, FIELD_READ, \
+ CONTAINER_READ, VALUE_READ, BOOL_READ, CompactType
+from thrift.protocol.TProtocol import TProtocolException
+from thrift.transport import TTransport, TZlibTransport
+
+
+class TAsyncioBaseTransport(TTransport.TTransportBase):
+ @asyncio.coroutine
+ def readAll(self, sz):
+ buff = b''
+ while sz:
+ chunk = yield from self.read(sz)
+ sz -= len(chunk)
+ buff += chunk
+
+ if len(chunk) == 0:
+ raise EOFError()
+
+ return buff
+
+
+class TAsyncioTransport(TAsyncioBaseTransport):
+ """An abstract transport over asyncio streams"""
+ def __init__(self, reader, writer):
+ self._reader = reader
+ self._writer = writer
+ self._wbuf = BytesIO()
+ self._logger = logging.getLogger("TAsyncioTransport")
+
+ @classmethod
+ @asyncio.coroutine
+ def connect(cls, host, port, loop=None, ssl=False):
+ reader, writer = yield from asyncio.open_connection(
+ host, port, loop=loop, ssl=ssl)
+ return cls(reader, writer)
+
+ def write(self, buf):
+ try:
+ self._wbuf.write(buf)
+ except Exception as e:
+ # reset wbuf so it doesn't contain a partial function call
+ self._wbuf.seek(0)
+ raise e from None
+
+ @asyncio.coroutine
+ def flush(self):
+ wbuf = self._wbuf
+ size = wbuf.tell()
+ if self._logger.isEnabledFor(logging.DEBUG):
+ self._logger.debug('writing %s of size %d: %s',
+ size, 'frame' if self.framed else "data",
+ wbuf.getvalue()[:size])
+ data = self._get_flushed_data()
+ self._writer.write(data)
+ wbuf.seek(0)
+ yield from self._writer.drain()
+
+ def close(self):
+ self._reader.feed_eof()
+ self._writer.close()
+
+
+class TAsyncioBufferedTransport(TAsyncioTransport):
+ """A buffered transport over asyncio streams"""
+
+ @asyncio.coroutine
+ def read(self, sz):
+ return (yield from self._reader.read(sz))
+
+ def _get_flushed_data(self):
+ wbuf = self._wbuf
+ size = wbuf.tell()
+ if size < 1024:
+ return wbuf.getvalue()[:size]
+ return memoryview(wbuf.getvalue())[:size]
+
+
+class TAsyncioFramedTransport(TAsyncioTransport):
+ """A buffered transport over an asyncio stream"""
+ def __init__(self, reader, writer):
+ super(TAsyncioFramedTransport, self).__init__(reader, writer)
+ self._rbuf = BytesIO()
+
+ @asyncio.coroutine
+ def read(self, sz):
+ ret = self._rbuf.read(sz)
+ if ret:
+ return ret
+ yield from self.readFrame()
+ return self._rbuf.read(sz)
+
+ @asyncio.coroutine
+ def readFrame(self):
+ buff = yield from self._reader.readexactly(4)
+ sz, = unpack('!i', buff)
+ self._rbuf = BytesIO((yield from self._reader.readexactly(sz)))
+
+ def _get_flushed_data(self):
+ wbuf = self._wbuf
+ size = wbuf.tell()
+ return pack("!i", size) + wbuf.getvalue()[:size]
+
+
+class TAsyncioZlibTransport(TZlibTransport.TZlibTransport,
+ TAsyncioBaseTransport):
+ """Class that wraps an asyncio-friendly transport with zlib, compressing
+ writes and decompresses reads, using the python standard
+ library zlib module.
+ """
+
+ @asyncio.coroutine
+ def read(self, sz):
+ """Read up to sz bytes from the decompressed bytes buffer, and
+ read from the underlying transport if the decompression
+ buffer is empty.
+ """
+ ret = self._rbuf.read(sz)
+ if len(ret) > 0:
+ return ret
+ # keep reading from transport until something comes back
+ while True:
+ if (yield from self.readComp(sz)):
+ break
+ ret = self._rbuf.read(sz)
+ return ret
+
+ @asyncio.coroutine
+ def readComp(self, sz):
+ """Read compressed data from the underlying transport, then
+ decompress it and append it to the internal StringIO read buffer
+ """
+ zbuf = yield from self._trans.read(sz)
+ return self._readComp(zbuf)
+
+ @asyncio.coroutine
+ def flush(self):
+ """Flush any queued up data in the write buffer and ensure the
+ compression buffer is flushed out to the underlying transport
+ """
+ super(TAsyncioZlibTransport, self).flush()
+ # flush() in the base class is effectively a no-op
+ yield from self._trans.flush()
+
+ @asyncio.coroutine
+ def cstringio_refill(self, partialread, reqlen):
+ """Implement the CReadableTransport interface for refill"""
+ retstring = partialread
+ if reqlen < self.DEFAULT_BUFFSIZE:
+ retstring += yield from self.read(self.DEFAULT_BUFFSIZE)
+ while len(retstring) < reqlen:
+ retstring += yield from self.read(reqlen - len(retstring))
+ self._rbuf = BytesIO(retstring)
+ return self._rbuf
+
+
+class TAsyncioBinaryProtocol(TBinaryProtocol):
+
+ """Binary implementation of the Thrift protocol driver for asyncio."""
+
+ _fast_encode = None
+ _fast_decode = None
+
+ @asyncio.coroutine
+ def readMessageBegin(self):
+ sz = yield from self.readI32()
+ if sz < 0:
+ version = sz & TBinaryProtocol.VERSION_MASK
+ if version != TBinaryProtocol.VERSION_1:
+ raise TProtocolException(
+ type=TProtocolException.BAD_VERSION,
+ message='Bad version in readMessageBegin: %d' % (sz))
+ type = sz & TBinaryProtocol.TYPE_MASK
+ name = yield from self.readString()
+ seqid = yield from self.readI32()
+ else:
+ if self.strictRead:
+ raise TProtocolException(type=TProtocolException.BAD_VERSION,
+ message='No protocol version header')
+ name = yield from self.trans.readAll(sz)
+ type = yield from self.readByte()
+ seqid = yield from self.readI32()
+ return (name, type, seqid)
+
+ @asyncio.coroutine
+ def readMessageEnd(self):
+ pass
+
+ @asyncio.coroutine
+ def readStructBegin(self):
+ pass
+
+ @asyncio.coroutine
+ def readStructEnd(self):
+ pass
+
+ @asyncio.coroutine
+ def readFieldBegin(self):
+ type = yield from self.readByte()
+ if type == TType.STOP:
+ return (None, type, 0)
+ id = yield from self.readI16()
+ return (None, type, id)
+
+ @asyncio.coroutine
+ def readFieldEnd(self):
+ pass
+
+ @asyncio.coroutine
+ def readMapBegin(self):
+ ktype = yield from self.readByte()
+ vtype = yield from self.readByte()
+ size = yield from self.readI32()
+ self._check_container_length(size)
+ return (ktype, vtype, size)
+
+ @asyncio.coroutine
+ def readMapEnd(self):
+ pass
+
+ @asyncio.coroutine
+ def readListBegin(self):
+ etype = yield from self.readByte()
+ size = yield from self.readI32()
+ self._check_container_length(size)
+ return (etype, size)
+
+ @asyncio.coroutine
+ def readListEnd(self):
+ pass
+
+ @asyncio.coroutine
+ def readSetBegin(self):
+ etype = yield from self.readByte()
+ size = yield from self.readI32()
+ self._check_container_length(size)
+ return (etype, size)
+
+ @asyncio.coroutine
+ def readSetEnd(self):
+ pass
+
+ @asyncio.coroutine
+ def readBool(self):
+ byte = yield from self.readByte()
+ if byte == 0:
+ return False
+ return True
+
+ @asyncio.coroutine
+ def readByte(self):
+ buff = yield from self.trans.readAll(1)
+ val, = unpack('!b', buff)
+ return val
+
+ @asyncio.coroutine
+ def readI16(self):
+ buff = yield from self.trans.readAll(2)
+ val, = unpack('!h', buff)
+ return val
+
+ @asyncio.coroutine
+ def readI32(self):
+ buff = yield from self.trans.readAll(4)
+ val, = unpack('!i', buff)
+ return val
+
+ @asyncio.coroutine
+ def readI64(self):
+ buff = yield from self.trans.readAll(8)
+ val, = unpack('!q', buff)
+ return val
+
+ @asyncio.coroutine
+ def readDouble(self):
+ buff = yield from self.trans.readAll(8)
+ val, = unpack('!d', buff)
+ return val
+
+ @asyncio.coroutine
+ def readBinary(self):
+ size = yield from self.readI32()
+ self._check_string_length(size)
+ s = yield from self.trans.readAll(size)
+ return s
+
+ @asyncio.coroutine
+ def readString(self):
+ return (yield from self.readBinary()).decode("utf-8")
+
+ @asyncio.coroutine
+ def skip(self, type):
+ if type == TType.STOP:
+ return
+ elif type == TType.BOOL:
+ yield from self.readBool()
+ elif type == TType.BYTE:
+ yield from self.readByte()
+ elif type == TType.I16:
+ yield from self.readI16()
+ elif type == TType.I32:
+ yield from self.readI32()
+ elif type == TType.I64:
+ yield from self.readI64()
+ elif type == TType.DOUBLE:
+ yield from self.readDouble()
+ elif type == TType.FLOAT:
+ yield from self.readFloat()
+ elif type == TType.STRING:
+ yield from self.readString()
+ elif type == TType.STRUCT:
+ name = yield from self.readStructBegin()
+ while True:
+ (name, type, id) = yield from self.readFieldBegin()
+ if type == TType.STOP:
+ break
+ yield from self.skip(type)
+ yield from self.readFieldEnd()
+ yield from self.readStructEnd()
+ elif type == TType.MAP:
+ (ktype, vtype, size) = yield from self.readMapBegin()
+ for i in range(size):
+ yield from self.skip(ktype)
+ yield from self.skip(vtype)
+ yield from self.readMapEnd()
+ elif type == TType.SET:
+ (etype, size) = yield from self.readSetBegin()
+ for i in range(size):
+ yield from self.skip(etype)
+ yield from self.readSetEnd()
+ elif type == TType.LIST:
+ (etype, size) = yield from self.readListBegin()
+ for i in range(size):
+ yield from self.skip(etype)
+ yield from self.readListEnd()
+
+
+class TAsyncioBinaryProtocolFactory(TBinaryProtocolFactory):
+ def getProtocol(self, trans):
+ return TAsyncioBinaryProtocol(trans, self.strictRead, self.strictWrite)
+
+
+class TAsyncioCompactProtocol(TCompactProtocol):
+ """Compact implementation of the Thrift protocol driver with asyncio."""
+
+ @asyncio.coroutine
+ def readString(self):
+ return (yield from self.readBinary()).decode("utf-8")
+
+ @asyncio.coroutine
+ def readFieldBegin(self):
+ assert self.state == FIELD_READ, self.state
+ type = yield from self.__readUByte()
+ if type & 0x0f == TType.STOP:
+ return (None, 0, 0)
+ delta = type >> 4
+ if delta == 0:
+ fid = yield from self.__readI16()
+ else:
+ fid = self._last_fid + delta
+ self._last_fid = fid
+ type = type & 0x0f
+ if type == CompactType.TRUE:
+ self.state = BOOL_READ
+ self._bool_value = True
+ elif type == CompactType.FALSE:
+ self.state = BOOL_READ
+ self._bool_value = False
+ else:
+ self.state = VALUE_READ
+ return (None, self._getTType(type), fid)
+
+ @asyncio.coroutine
+ def readFieldEnd(self):
+ super(TAsyncioCompactProtocol, self).readFieldEnd()
+
+ @asyncio.coroutine
+ def __readUByte(self):
+ result, = unpack('!B', (yield from self.trans.readAll(1)))
+ return result
+
+ @asyncio.coroutine
+ def __readByte(self):
+ result, = unpack('!b', (yield from self.trans.readAll(1)))
+ return result
+
+ @asyncio.coroutine
+ def __readVarint(self):
+ result = 0
+ shift = 0
+ while True:
+ x = yield from self.trans.readAll(1)
+ byte = ord(x)
+ result |= (byte & 0x7f) << shift
+ if byte >> 7 == 0:
+ return result
+ shift += 7
+
+ @asyncio.coroutine
+ def __readZigZag(self):
+ return fromZigZag((yield from self.__readVarint()))
+
+ @asyncio.coroutine
+ def __readSize(self):
+ result = yield from self.__readVarint()
+ if result < 0:
+ raise TProtocolException("Length < 0")
+ return result
+
+ @asyncio.coroutine
+ def readMessageBegin(self):
+ assert self.state == CLEAR
+ proto_id = yield from self.__readUByte()
+ if proto_id != self.PROTOCOL_ID:
+ raise TProtocolException(TProtocolException.BAD_VERSION,
+ 'Bad protocol id in the message: %d' % proto_id)
+ ver_type = yield from self.__readUByte()
+ type = (ver_type >> self.TYPE_SHIFT_AMOUNT) & self.TYPE_BITS
+ version = ver_type & self.VERSION_MASK
+ if version != self.VERSION:
+ raise TProtocolException(TProtocolException.BAD_VERSION,
+ 'Bad version: %d (expect %d)' % (version, self.VERSION))
+ seqid = yield from self.__readVarint()
+ name = (yield from self.__readBinary()).decode("utf-8")
+ return (name, type, seqid)
+
+ @asyncio.coroutine
+ def readMessageEnd(self):
+ super(TAsyncioCompactProtocol, self).readMessageEnd()
+
+ @asyncio.coroutine
+ def readStructBegin(self):
+ super(TAsyncioCompactProtocol, self).readStructBegin()
+
+ @asyncio.coroutine
+ def readStructEnd(self):
+ super(TAsyncioCompactProtocol, self).readStructEnd()
+
+ @asyncio.coroutine
+ def readCollectionBegin(self):
+ assert self.state in (VALUE_READ, CONTAINER_READ), self.state
+ size_type = yield from self.__readUByte()
+ size = size_type >> 4
+ type = self._getTType(size_type)
+ if size == 15:
+ size = yield from self.__readSize()
+ self._check_container_length(size)
+ self._containers.append(self.state)
+ self.state = CONTAINER_READ
+ return type, size
+ readSetBegin = readCollectionBegin
+ readListBegin = readCollectionBegin
+
+ @asyncio.coroutine
+ def readMapBegin(self):
+ assert self.state in (VALUE_READ, CONTAINER_READ), self.state
+ size = yield from self.__readSize()
+ self._check_container_length(size)
+ types = 0
+ if size > 0:
+ types = yield from self.__readUByte()
+ vtype = self._getTType(types)
+ ktype = self._getTType(types >> 4)
+ self._containers.append(self.state)
+ self.state = CONTAINER_READ
+ return (ktype, vtype, size)
+
+ @asyncio.coroutine
+ def readCollectionEnd(self):
+ super(TAsyncioCompactProtocol, self).readCollectionEnd()
+ readSetEnd = readCollectionEnd
+ readListEnd = readCollectionEnd
+ readMapEnd = readCollectionEnd
+
+ @asyncio.coroutine
+ def readBool(self):
+ return super(TAsyncioCompactProtocol, self).readBool()
+
+ readByte = reader(__readByte)
+ __readI16 = __readZigZag
+ readI16 = reader(__readZigZag)
+ readI32 = reader(__readZigZag)
+ readI64 = reader(__readZigZag)
+
+ @asyncio.coroutine
+ @reader
+ def readDouble(self):
+ buff = yield from self.trans.readAll(8)
+ val, = unpack('<d', buff)
+ return val
+
+ @asyncio.coroutine
+ def __readBinary(self):
+ size = yield from self.__readSize()
+ self._check_string_length(size)
+ return (yield from self.trans.readAll(size))
+ readBinary = reader(__readBinary)
+
+
+class TAsyncioCompactProtocolFactory(TCompactProtocolFactory):
+ def getProtocol(self, trans):
+ return TAsyncioCompactProtocol(trans,
+ self.string_length_limit,
+ self.container_length_limit)
+
+
+class TAsyncioApplicationException(TApplicationException):
+ @asyncio.coroutine
+ def read(self, iprot):
+ yield from iprot.readStructBegin()
+ while True:
+ (fname, ftype, fid) = yield from iprot.readFieldBegin()
+ if ftype == TType.STOP:
+ break
+ if fid == 1:
+ if ftype == TType.STRING:
+ message = yield from iprot.readString()
+ if isinstance(message, bytes):
+ try:
+ message = message.decode('utf-8')
+ except UnicodeDecodeError:
+ pass
+ self.message = message
+ else:
+ yield from iprot.skip(ftype)
+ elif fid == 2:
+ if ftype == TType.I32:
+ self.type = yield from iprot.readI32()
+ else:
+ yield from iprot.skip(ftype)
+ else:
+ yield from iprot.skip(ftype)
+ yield from iprot.readFieldEnd()
+ yield from iprot.readStructEnd()
diff --git a/lib/py/src/protocol/TCompactProtocol.py b/lib/py/src/protocol/TCompactProtocol.py
index 16fd9be28c..dad8940e1f 100644
--- a/lib/py/src/protocol/TCompactProtocol.py
+++ b/lib/py/src/protocol/TCompactProtocol.py
@@ -132,11 +132,11 @@ def __init__(self, trans,
container_length_limit=None):
TProtocolBase.__init__(self, trans)
self.state = CLEAR
- self.__last_fid = 0
- self.__bool_fid = None
- self.__bool_value = None
- self.__structs = []
- self.__containers = []
+ self._last_fid = 0
+ self._bool_fid = None
+ self._bool_value = None
+ self._structs = []
+ self._containers = []
self.string_length_limit = string_length_limit
self.container_length_limit = container_length_limit
@@ -163,31 +163,31 @@ def writeMessageEnd(self):
def writeStructBegin(self, name):
assert self.state in (CLEAR, CONTAINER_WRITE, VALUE_WRITE), self.state
- self.__structs.append((self.state, self.__last_fid))
+ self._structs.append((self.state, self._last_fid))
self.state = FIELD_WRITE
- self.__last_fid = 0
+ self._last_fid = 0
def writeStructEnd(self):
assert self.state == FIELD_WRITE
- self.state, self.__last_fid = self.__structs.pop()
+ self.state, self._last_fid = self._structs.pop()
def writeFieldStop(self):
self.__writeByte(0)
def __writeFieldHeader(self, type, fid):
- delta = fid - self.__last_fid
+ delta = fid - self._last_fid
if 0 < delta <= 15:
self.__writeUByte(delta << 4 | type)
else:
self.__writeByte(type)
self.__writeI16(fid)
- self.__last_fid = fid
+ self._last_fid = fid
def writeFieldBegin(self, name, type, fid):
assert self.state == FIELD_WRITE, self.state
if type == TType.BOOL:
self.state = BOOL_WRITE
- self.__bool_fid = fid
+ self._bool_fid = fid
else:
self.state = VALUE_WRITE
self.__writeFieldHeader(CTYPES[type], fid)
@@ -215,7 +215,7 @@ def writeCollectionBegin(self, etype, size):
else:
self.__writeUByte(0xf0 | CTYPES[etype])
self.__writeSize(size)
- self.__containers.append(self.state)
+ self._containers.append(self.state)
self.state = CONTAINER_WRITE
writeSetBegin = writeCollectionBegin
writeListBegin = writeCollectionBegin
@@ -227,12 +227,12 @@ def writeMapBegin(self, ktype, vtype, size):
else:
self.__writeSize(size)
self.__writeUByte(CTYPES[ktype] << 4 | CTYPES[vtype])
- self.__containers.append(self.state)
+ self._containers.append(self.state)
self.state = CONTAINER_WRITE
def writeCollectionEnd(self):
assert self.state == CONTAINER_WRITE, self.state
- self.state = self.__containers.pop()
+ self.state = self._containers.pop()
writeMapEnd = writeCollectionEnd
writeSetEnd = writeCollectionEnd
writeListEnd = writeCollectionEnd
@@ -243,7 +243,7 @@ def writeBool(self, bool):
ctype = CompactType.TRUE
else:
ctype = CompactType.FALSE
- self.__writeFieldHeader(ctype, self.__bool_fid)
+ self.__writeFieldHeader(ctype, self._bool_fid)
elif self.state == CONTAINER_WRITE:
if bool:
self.__writeByte(CompactType.TRUE)
@@ -281,18 +281,18 @@ def readFieldBegin(self):
if delta == 0:
fid = self.__readI16()
else:
- fid = self.__last_fid + delta
- self.__last_fid = fid
+ fid = self._last_fid + delta
+ self._last_fid = fid
type = type & 0x0f
if type == CompactType.TRUE:
self.state = BOOL_READ
- self.__bool_value = True
+ self._bool_value = True
elif type == CompactType.FALSE:
self.state = BOOL_READ
- self.__bool_value = False
+ self._bool_value = False
else:
self.state = VALUE_READ
- return (None, self.__getTType(type), fid)
+ return (None, self._getTType(type), fid)
def readFieldEnd(self):
assert self.state in (VALUE_READ, BOOL_READ), self.state
@@ -336,27 +336,27 @@ def readMessageBegin(self):
def readMessageEnd(self):
assert self.state == CLEAR
- assert len(self.__structs) == 0
+ assert len(self._structs) == 0
def readStructBegin(self):
assert self.state in (CLEAR, CONTAINER_READ, VALUE_READ), self.state
- self.__structs.append((self.state, self.__last_fid))
+ self._structs.append((self.state, self._last_fid))
self.state = FIELD_READ
- self.__last_fid = 0
+ self._last_fid = 0
def readStructEnd(self):
assert self.state == FIELD_READ
- self.state, self.__last_fid = self.__structs.pop()
+ self.state, self._last_fid = self._structs.pop()
def readCollectionBegin(self):
assert self.state in (VALUE_READ, CONTAINER_READ), self.state
size_type = self.__readUByte()
size = size_type >> 4
- type = self.__getTType(size_type)
+ type = self._getTType(size_type)
if size == 15:
size = self.__readSize()
self._check_container_length(size)
- self.__containers.append(self.state)
+ self._containers.append(self.state)
self.state = CONTAINER_READ
return type, size
readSetBegin = readCollectionBegin
@@ -369,22 +369,22 @@ def readMapBegin(self):
types = 0
if size > 0:
types = self.__readUByte()
- vtype = self.__getTType(types)
- ktype = self.__getTType(types >> 4)
- self.__containers.append(self.state)
+ vtype = self._getTType(types)
+ ktype = self._getTType(types >> 4)
+ self._containers.append(self.state)
self.state = CONTAINER_READ
return (ktype, vtype, size)
def readCollectionEnd(self):
assert self.state == CONTAINER_READ, self.state
- self.state = self.__containers.pop()
+ self.state = self._containers.pop()
readSetEnd = readCollectionEnd
readListEnd = readCollectionEnd
readMapEnd = readCollectionEnd
def readBool(self):
if self.state == BOOL_READ:
- return self.__bool_value == CompactType.TRUE
+ return self._bool_value == CompactType.TRUE
elif self.state == CONTAINER_READ:
return self.__readByte() == CompactType.TRUE
else:
@@ -409,7 +409,7 @@ def __readBinary(self):
return self.trans.readAll(size)
readBinary = reader(__readBinary)
- def __getTType(self, byte):
+ def _getTType(self, byte):
return TTYPES[byte & 0x0f]
diff --git a/lib/py/src/transport/TZlibTransport.py b/lib/py/src/transport/TZlibTransport.py
index e848579246..fe323c8d4d 100644
--- a/lib/py/src/transport/TZlibTransport.py
+++ b/lib/py/src/transport/TZlibTransport.py
@@ -86,10 +86,10 @@ def __init__(self, trans, compresslevel=9):
from 0 (no compression) to 9 (best compression). Default is 9.
@type compresslevel: int
"""
- self.__trans = trans
+ self._trans = trans
self.compresslevel = compresslevel
- self.__rbuf = BufferIO()
- self.__wbuf = BufferIO()
+ self._rbuf = BufferIO()
+ self._wbuf = BufferIO()
self._init_zlib()
self._init_stats()
@@ -97,8 +97,8 @@ def _reinit_buffers(self):
"""Internal method to initialize/reset the internal StringIO objects
for read and write buffers.
"""
- self.__rbuf = BufferIO()
- self.__wbuf = BufferIO()
+ self._rbuf = BufferIO()
+ self._wbuf = BufferIO()
def _init_stats(self):
"""Internal method to reset the internal statistics counters
@@ -157,68 +157,60 @@ def getCompSavings(self):
def isOpen(self):
"""Return the underlying transport's open status"""
- return self.__trans.isOpen()
+ return self._trans.isOpen()
def open(self):
"""Open the underlying transport"""
self._init_stats()
- return self.__trans.open()
+ return self._trans.open()
def listen(self):
"""Invoke the underlying transport's listen() method"""
- self.__trans.listen()
+ self._trans.listen()
def accept(self):
"""Accept connections on the underlying transport"""
- return self.__trans.accept()
+ return self._trans.accept()
def close(self):
"""Close the underlying transport,"""
self._reinit_buffers()
self._init_zlib()
- return self.__trans.close()
+ return self._trans.close()
def read(self, sz):
"""Read up to sz bytes from the decompressed bytes buffer, and
read from the underlying transport if the decompression
buffer is empty.
"""
- ret = self.__rbuf.read(sz)
+ ret = self._rbuf.read(sz)
if len(ret) > 0:
return ret
# keep reading from transport until something comes back
while True:
if self.readComp(sz):
break
- ret = self.__rbuf.read(sz)
+ ret = self._rbuf.read(sz)
return ret
def readComp(self, sz):
"""Read compressed data from the underlying transport, then
decompress it and append it to the internal StringIO read buffer
"""
- zbuf = self.__trans.read(sz)
- zbuf = self._zcomp_read.unconsumed_tail + zbuf
- buf = self._zcomp_read.decompress(zbuf)
- self.bytes_in += len(zbuf)
- self.bytes_in_comp += len(buf)
- old = self.__rbuf.read()
- self.__rbuf = BufferIO(old + buf)
- if len(old) + len(buf) == 0:
- return False
- return True
+ zbuf = self._trans.read(sz)
+ return self._readComp(zbuf)
def write(self, buf):
"""Write some bytes, putting them into the internal write
buffer for eventual compression.
"""
- self.__wbuf.write(buf)
+ self._wbuf.write(buf)
def flush(self):
"""Flush any queued up data in the write buffer and ensure the
compression buffer is flushed out to the underlying transport
"""
- wout = self.__wbuf.getvalue()
+ wout = self._wbuf.getvalue()
if len(wout) > 0:
zbuf = self._zcomp_write.compress(wout)
self.bytes_out += len(wout)
@@ -228,14 +220,14 @@ def flush(self):
ztail = self._zcomp_write.flush(zlib.Z_SYNC_FLUSH)
self.bytes_out_comp += len(ztail)
if (len(zbuf) + len(ztail)) > 0:
- self.__wbuf = BufferIO()
- self.__trans.write(zbuf + ztail)
- self.__trans.flush()
+ self._wbuf = BufferIO()
+ self._trans.write(zbuf + ztail)
+ self._trans.flush()
@property
def cstringio_buf(self):
"""Implement the CReadableTransport interface"""
- return self.__rbuf
+ return self._rbuf
def cstringio_refill(self, partialread, reqlen):
"""Implement the CReadableTransport interface for refill"""
@@ -244,5 +236,17 @@ def cstringio_refill(self, partialread, reqlen):
retstring += self.read(self.DEFAULT_BUFFSIZE)
while len(retstring) < reqlen:
retstring += self.read(reqlen - len(retstring))
- self.__rbuf = BufferIO(retstring)
- return self.__rbuf
+ self._rbuf = BufferIO(retstring)
+ return self._rbuf
+
+ def _readComp(self, zbuf):
+ """Decompress data read from the underlying transport, then
+ append it to the internal StringIO read buffer
+ """
+ zbuf = self._zcomp_read.unconsumed_tail + zbuf
+ buf = self._zcomp_read.decompress(zbuf)
+ self.bytes_in += len(zbuf)
+ self.bytes_in_comp += len(buf)
+ old = self._rbuf.read()
+ self._rbuf = BufferIO(old + buf)
+ return (len(old) + len(buf)) != 0
diff --git a/test/py/Makefile.am b/test/py/Makefile.am
index f105737cd2..39c11a9929 100644
--- a/test/py/Makefile.am
+++ b/test/py/Makefile.am
@@ -22,6 +22,12 @@ THRIFT = $(top_builddir)/compiler/cpp/thrift
py_unit_tests = RunClientServer.py
+if WITH_PY3
+
+py_unit_tests += RunAsyncioClientServer.py
+
+endif
+
thrift_gen = \
gen-py/ThriftTest/__init__.py \
gen-py/DebugProtoTest/__init__.py \
@@ -36,13 +42,16 @@ thrift_gen = \
gen-py-dynamic/ThriftTest/__init__.py \
gen-py-dynamic/DebugProtoTest/__init__.py \
gen-py-dynamicslots/ThriftTest/__init__.py \
- gen-py-dynamicslots/DebugProtoTest/__init__.py
+ gen-py-dynamicslots/DebugProtoTest/__init__.py \
+ gen-py-asyncio/ThriftTest/__init__.py \
+ gen-py-asyncio/DebugProtoTest/__init__.py
precross: $(thrift_gen)
BUILT_SOURCES = $(thrift_gen)
helper_scripts= \
TestClient.py \
+ TestAsyncioClient.py \
TestServer.py
check_SCRIPTS= \
@@ -80,5 +89,9 @@ gen-py-dynamicslots/%/__init__.py: ../%.thrift $(THRIFT)
test -d gen-py-dynamicslots || $(MKDIR_P) gen-py-dynamicslots
$(THRIFT) --gen py:dynamic,slots -out gen-py-dynamicslots $<
+gen-py-asyncio/%/__init__.py: ../%.thrift $(THRIFT)
+ test -d gen-py-asyncio || $(MKDIR_P) gen-py-asyncio
+ $(THRIFT) --gen py:asyncio -out gen-py-asyncio $<
+
clean-local:
- $(RM) -r gen-py gen-py-slots gen-py-default gen-py-oldstyle gen-py-no_utf8strings gen-py-dynamic gen-py-dynamicslots
+ $(RM) -r gen-py gen-py-slots gen-py-default gen-py-oldstyle gen-py-no_utf8strings gen-py-dynamic gen-py-dynamicslots gen-py-asyncio
diff --git a/test/py/RunAsyncioClientServer.py b/test/py/RunAsyncioClientServer.py
new file mode 100755
index 0000000000..06f3593221
--- /dev/null
+++ b/test/py/RunAsyncioClientServer.py
@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import os
+import subprocess
+import sys
+
+
+def main():
+ basedir = os.path.dirname(__file__)
+ subprocess.call([
+ sys.executable, os.path.join(basedir, "RunClientServer.py"),
+ "--genpydirs", "asyncio", "--protos", "binary,compact",
+ "--scripts", ""])
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/test/py/RunClientServer.py b/test/py/RunClientServer.py
index c1443ec178..4c044f6b7d 100755
--- a/test/py/RunClientServer.py
+++ b/test/py/RunClientServer.py
@@ -105,7 +105,8 @@ def runServiceTest(libdir, genbase, genpydir, server_class, proto, port, use_zli
env = setup_pypath(libdir, os.path.join(genbase, genpydir))
# Build command line arguments
server_args = [sys.executable, relfile('TestServer.py')]
- cli_args = [sys.executable, relfile('TestClient.py')]
+ cli_args = [sys.executable, relfile(
+ 'Test%sClient.py' % ('Asyncio' if genpydir.endswith('asyncio') else ''))]
for which in (server_args, cli_args):
which.append('--protocol=%s' % proto) # accel, binary, compact or json
which.append('--port=%d' % port) # default to 9090
@@ -182,19 +183,21 @@ def ensureServerAlive():
class TestCases(object):
- def __init__(self, genbase, libdir, port, gendirs, servers, verbose):
+ def __init__(self, genbase, libdir, port, gendirs, servers, protos,
+ verbose):
self.genbase = genbase
self.libdir = libdir
self.port = port
self.verbose = verbose
self.gendirs = gendirs
self.servers = servers
+ self.protos = protos
def default_conf(self):
return {
'gendir': self.gendirs[0],
'server': self.servers[0],
- 'proto': PROTOS[0],
+ 'proto': self.protos[0],
'zlib': False,
'ssl': False,
}
@@ -211,6 +214,9 @@ def run(self, conf, test_count):
# skip any servers that don't work with SSL
if with_ssl and try_server in SKIP_SSL:
return False
+ if genpydir.endswith('asyncio') and try_server == 'THttpServer':
+ # FIXME: remove this when THttpClient is ported to asyncio
+ return False
if self.verbose > 0:
print('\nTest run #%d: (includes %s) Server=%s, Proto=%s, zlib=%s, SSL=%s'
% (test_count, genpydir, try_server, try_proto, with_zlib, with_ssl))
@@ -233,7 +239,10 @@ def run_all_tests(self):
test_count = 0
for try_server in self.servers:
for genpydir in self.gendirs:
- for try_proto in PROTOS:
+ if genpydir.endswith('asyncio') and try_server == 'THttpServer':
+ # FIXME: remove this when THttpClient is ported to asyncio
+ continue
+ for try_proto in self.protos:
for with_zlib in (False, True):
# skip any servers that don't work with the Zlib transport
if with_zlib and try_server in SKIP_ZLIB:
@@ -261,6 +270,10 @@ def main():
help='directory extensions for generated code, used as suffixes for \"gen-py-*\" added sys.path for individual tests')
parser.add_option("--port", type="int", dest="port", default=9090,
help="port number for server to listen on")
+ parser.add_option('--scripts', type='string', dest='scripts',
+ default=','.join(SCRIPTS), help='Scripts to be tested')
+ parser.add_option('--protos', type='string', dest='protos',
+ default=','.join(PROTOS), help='Protocols to be tested')
parser.add_option('-v', '--verbose', action="store_const",
dest="verbose", const=2,
help="verbose output")
@@ -276,7 +289,10 @@ def main():
generated_dirs = []
for gp_dir in options.genpydirs.split(','):
- generated_dirs.append('gen-py-%s' % (gp_dir))
+ if gp_dir:
+ generated_dirs.append('gen-py-%s' % (gp_dir))
+ scripts = [s for s in options.scripts.split(',') if s]
+ protos = [p for p in options.protos.split(',') if p]
# commandline permits a single class name to be specified to override SERVERS=[...]
servers = default_servers()
@@ -287,23 +303,24 @@ def main():
print('Unavailable server type "%s", please choose one of: %s' % (args[0], servers))
sys.exit(0)
- tests = TestCases(options.gen_base, options.libdir, options.port, generated_dirs, servers, options.verbose)
+ tests = TestCases(options.gen_base, options.libdir, options.port,
+ generated_dirs, servers, protos, options.verbose)
# run tests without a client/server first
print('----------------')
print(' Executing individual test scripts with various generated code directories')
print(' Directories to be tested: ' + ', '.join(generated_dirs))
- print(' Scripts to be tested: ' + ', '.join(SCRIPTS))
+ print(' Scripts to be tested: ' + ', '.join(scripts))
print('----------------')
for genpydir in generated_dirs:
- for script in SCRIPTS:
+ for script in scripts:
runScriptTest(options.libdir, options.gen_base, genpydir, script)
print('----------------')
print(' Executing Client/Server tests with various generated code directories')
print(' Servers to be tested: ' + ', '.join(servers))
print(' Directories to be tested: ' + ', '.join(generated_dirs))
- print(' Protocols to be tested: ' + ', '.join(PROTOS))
+ print(' Protocols to be tested: ' + ', '.join(protos))
print(' Options to be tested: ZLIB(yes/no), SSL(yes/no)')
print('----------------')
@@ -312,7 +329,7 @@ def main():
else:
tests.test_feature('gendir', generated_dirs)
tests.test_feature('server', servers)
- tests.test_feature('proto', PROTOS)
+ tests.test_feature('proto', protos)
tests.test_feature('zlib', [False, True])
tests.test_feature('ssl', [False, True])
diff --git a/test/py/TestAsyncioClient.py b/test/py/TestAsyncioClient.py
new file mode 100755
index 0000000000..42cc1c0521
--- /dev/null
+++ b/test/py/TestAsyncioClient.py
@@ -0,0 +1,350 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import asyncio
+import os
+import ssl
+import sys
+import time
+import unittest
+from optparse import OptionParser
+
+from util import local_libpath
+
+SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
+
+
+def async_test(f):
+ def wrapper(*args, **kwargs):
+ coro = asyncio.coroutine(f)
+ future = coro(*args, **kwargs)
+ loop = asyncio.get_event_loop()
+ loop.run_until_complete(future)
+ return wrapper
+
+
+class AbstractTest(unittest.TestCase):
+ @async_test
+ def setUp(self):
+ if options.ssl:
+ ssl_ctx = ssl.create_default_context()
+ ssl_ctx.check_hostname = False
+ ssl_ctx.verify_mode = ssl.CERT_NONE
+ else:
+ ssl_ctx = None
+
+ if options.trans == 'framed':
+ self.transport = \
+ yield from TAsyncio.TAsyncioFramedTransport.connect(
+ options.host, options.port, ssl=ssl_ctx)
+ elif options.trans == 'buffered':
+ self.transport = \
+ yield from TAsyncio.TAsyncioBufferedTransport.connect(
+ options.host, options.port, ssl=ssl_ctx)
+ elif options.trans == '':
+ raise AssertionError('Unknown --transport option: %s' % options.trans)
+ if options.zlib:
+ self.transport = TAsyncio.TAsyncioZlibTransport(self.transport, 9)
+ self.transport.open()
+ protocol = self.get_protocol(self.transport)
+ self.client = ThriftTest.Client(protocol)
+
+ def tearDown(self):
+ self.transport.close()
+
+ @async_test
+ def testVoid(self):
+ print('testVoid')
+ yield from self.client.testVoid()
+
+ @async_test
+ def testString(self):
+ print('testString')
+ self.assertEqual((yield from self.client.testString('Python' * 20)), 'Python' * 20)
+ self.assertEqual((yield from self.client.testString('')), '')
+ s1 = u'\b\t\n/\\\\\r{}:パイソン"'
+ s2 = u"""Afrikaans, Alemannisch, Aragonés, العربية, مصرى,
+ Asturianu, Aymar aru, Azərbaycan, Башҡорт, Boarisch, Žemaitėška,
+ Беларуская, Беларуская (тарашкевіца), Български, Bamanankan,
+ বাংলা, Brezhoneg, Bosanski, Català, Mìng-dĕ̤ng-ngṳ̄, Нохчийн,
+ Cebuano, ᏣᎳᎩ, Česky, Словѣ́ньскъ / ⰔⰎⰑⰂⰡⰐⰠⰔⰍⰟ, Чӑвашла, Cymraeg,
+ Dansk, Zazaki, ދިވެހިބަސް, Ελληνικά, Emiliàn e rumagnòl, English,
+ Esperanto, Español, Eesti, Euskara, فارسی, Suomi, Võro, Føroyskt,
+ Français, Arpetan, Furlan, Frysk, Gaeilge, 贛語, Gàidhlig, Galego,
+ Avañe'ẽ, ગુજરાતી, Gaelg, עברית, हिन्दी, Fiji Hindi, Hrvatski,
+ Kreyòl ayisyen, Magyar, Հայերեն, Interlingua, Bahasa Indonesia,
+ Ilokano, Ido, Íslenska, Italiano, 日本語, Lojban, Basa Jawa,
+ ქართული, Kongo, Kalaallisut, ಕನ್ನಡ, 한국어, Къарачай-Малкъар,
+ Ripoarisch, Kurdî, Коми, Kernewek, Кыргызча, Latina, Ladino,
+ Lëtzebuergesch, Limburgs, Lingála, ລາວ, Lietuvių, Latviešu, Basa
+ Banyumasan, Malagasy, Македонски, മലയാളം, मराठी, مازِرونی, Bahasa
+ Melayu, Nnapulitano, Nedersaksisch, नेपाल भाषा, Nederlands,
+ Norsk (nynorsk), Norsk (bokmål), Nouormand, Diné bizaad,
+ Occitan, Иронау, Papiamentu, Deitsch, Polski, پنجابی, پښتو,
+ Norfuk / Pitkern, Português, Runa Simi, Rumantsch, Romani, Română,
+ Русский, Саха тыла, Sardu, Sicilianu, Scots, Sámegiella, Simple
+ English, Slovenčina, Slovenščina, Српски / Srpski, Seeltersk,
+ Svenska, Kiswahili, தமிழ், తెలుగు, Тоҷикӣ, ไทย, Türkmençe, Tagalog,
+ Türkçe, Татарча/Tatarça, Українська, اردو, Tiếng Việt, Volapük,
+ Walon, Winaray, 吴语, isiXhosa, ייִדיש, Yorùbá, Zeêuws, 中文,
+ Bân-lâm-gú, 粵語"""
+ self.assertEqual((yield from self.client.testString(s1)), s1)
+ self.assertEqual((yield from self.client.testString(s2)), s2)
+
+ @async_test
+ def testBool(self):
+ print('testBool')
+ self.assertEqual((yield from self.client.testBool(True)), True)
+ self.assertEqual((yield from self.client.testBool(False)), False)
+
+ @async_test
+ def testByte(self):
+ print('testByte')
+ self.assertEqual((yield from self.client.testByte(63)), 63)
+ self.assertEqual((yield from self.client.testByte(-127)), -127)
+
+ @async_test
+ def testI32(self):
+ print('testI32')
+ self.assertEqual((yield from self.client.testI32(-1)), -1)
+ self.assertEqual((yield from self.client.testI32(0)), 0)
+
+ @async_test
+ def testI64(self):
+ print('testI64')
+ self.assertEqual((yield from self.client.testI64(1)), 1)
+ self.assertEqual((yield from self.client.testI64(-34359738368)), -34359738368)
+
+ @async_test
+ def testDouble(self):
+ print('testDouble')
+ self.assertEqual((yield from self.client.testDouble(-5.235098235)), -5.235098235)
+ self.assertEqual((yield from self.client.testDouble(0)), 0)
+ self.assertEqual((yield from self.client.testDouble(-1)), -1)
+ self.assertEqual((yield from self.client.testDouble(-0.000341012439638598279)), -0.000341012439638598279)
+
+ @async_test
+ def testBinary(self):
+ print('testBinary')
+ val = bytearray([i for i in range(0, 256)])
+ self.assertEqual(bytearray((yield from self.client.testBinary(bytes(val)))), val)
+
+ @async_test
+ def testStruct(self):
+ print('testStruct')
+ x = Xtruct()
+ x.string_thing = "Zero"
+ x.byte_thing = 1
+ x.i32_thing = -3
+ x.i64_thing = -5
+ y = yield from self.client.testStruct(x)
+ self.assertEqual(y, x)
+
+ @async_test
+ def testNest(self):
+ print('testNest')
+ inner = Xtruct(string_thing="Zero", byte_thing=1, i32_thing=-3, i64_thing=-5)
+ x = Xtruct2(struct_thing=inner, byte_thing=0, i32_thing=0)
+ y = yield from self.client.testNest(x)
+ self.assertEqual(y, x)
+
+ @async_test
+ def testMap(self):
+ print('testMap')
+ x = {0: 1, 1: 2, 2: 3, 3: 4, -1: -2}
+ y = yield from self.client.testMap(x)
+ self.assertEqual(y, x)
+
+ @async_test
+ def testSet(self):
+ print('testSet')
+ x = set([8, 1, 42])
+ y = yield from self.client.testSet(x)
+ self.assertEqual(y, x)
+
+ @async_test
+ def testList(self):
+ print('testList')
+ x = [1, 4, 9, -42]
+ y = yield from self.client.testList(x)
+ self.assertEqual(y, x)
+
+ @async_test
+ def testEnum(self):
+ print('testEnum')
+ x = Numberz.FIVE
+ y = yield from self.client.testEnum(x)
+ self.assertEqual(y, x)
+
+ @async_test
+ def testTypedef(self):
+ print('testTypedef')
+ x = 0xffffffffffffff # 7 bytes of 0xff
+ y = yield from self.client.testTypedef(x)
+ self.assertEqual(y, x)
+
+ @async_test
+ def testMapMap(self):
+ print('testMapMap')
+ x = {
+ -4: {-4: -4, -3: -3, -2: -2, -1: -1},
+ 4: {4: 4, 3: 3, 2: 2, 1: 1},
+ }
+ y = yield from self.client.testMapMap(42)
+ self.assertEqual(y, x)
+
+ @async_test
+ def testMulti(self):
+ print('testMulti')
+ xpected = Xtruct(string_thing='Hello2', byte_thing=74, i32_thing=0xff00ff, i64_thing=0xffffffffd0d0)
+ y = yield from self.client.testMulti(xpected.byte_thing,
+ xpected.i32_thing,
+ xpected.i64_thing,
+ {0: 'abc'},
+ Numberz.FIVE,
+ 0xf0f0f0)
+ self.assertEqual(y, xpected)
+
+ @async_test
+ def testException(self):
+ print('testException')
+ yield from self.client.testException('Safe')
+ try:
+ yield from self.client.testException('Xception')
+ self.fail("should have gotten exception")
+ except Xception as x:
+ self.assertEqual(x.errorCode, 1001)
+ self.assertEqual(x.message, 'Xception')
+ # TODO ensure same behavior for repr within generated python variants
+ # ensure exception's repr method works
+ # x_repr = repr(x)
+ # self.assertEqual(x_repr, 'Xception(errorCode=1001, message=\'Xception\')')
+
+ try:
+ yield from self.client.testException('TException')
+ self.fail("should have gotten exception")
+ except TException as x:
+ pass
+
+ # Should not throw
+ yield from self.client.testException('success')
+
+ @async_test
+ def testMultiException(self):
+ print('testMultiException')
+ try:
+ yield from self.client.testMultiException('Xception', 'ignore')
+ except Xception as ex:
+ self.assertEqual(ex.errorCode, 1001)
+ self.assertEqual(ex.message, 'This is an Xception')
+
+ try:
+ yield from self.client.testMultiException('Xception2', 'ignore')
+ except Xception2 as ex:
+ self.assertEqual(ex.errorCode, 2002)
+ self.assertEqual(ex.struct_thing.string_thing, 'This is an Xception2')
+
+ y = yield from self.client.testMultiException('success', 'foobar')
+ self.assertEqual(y.string_thing, 'foobar')
+
+ @async_test
+ def testOneway(self):
+ print('testOneway')
+ start = time.time()
+ yield from self.client.testOneway(1) # type is int, not float
+ end = time.time()
+ self.assertTrue(end - start < 1,
+ "oneway sleep took %f sec" % (end - start))
+
+ @async_test
+ def testOnewayThenNormal(self):
+ print('testOnewayThenNormal')
+ yield from self.client.testOneway(1) # type is int, not float
+ self.assertEqual((yield from self.client.testString('Python')), 'Python')
+
+
+class NormalBinaryTest(AbstractTest):
+ def get_protocol(self, transport):
+ return TAsyncio.TAsyncioBinaryProtocolFactory().getProtocol(transport)
+
+
+class CompactTest(AbstractTest):
+ def get_protocol(self, transport):
+ return TAsyncio.TAsyncioCompactProtocolFactory().getProtocol(transport)
+
+
+def suite():
+ suite = unittest.TestSuite()
+ loader = unittest.TestLoader()
+ if options.proto == 'binary': # look for --proto on cmdline
+ suite.addTest(loader.loadTestsFromTestCase(NormalBinaryTest))
+ elif options.proto == 'compact':
+ suite.addTest(loader.loadTestsFromTestCase(CompactTest))
+ else:
+ raise AssertionError('Unknown protocol given with --protocol: %s' % options.proto)
+ return suite
+
+
+class OwnArgsTestProgram(unittest.TestProgram):
+ def parseArgs(self, argv):
+ if args:
+ self.testNames = args
+ else:
+ self.testNames = ([self.defaultTest])
+ self.createTests()
+
+if __name__ == "__main__":
+ parser = OptionParser()
+ parser.add_option('--libpydir', type='string', dest='libpydir',
+ help='include this directory in sys.path for locating library code')
+ parser.add_option('--genpydir', type='string', dest='genpydir',
+ help='include this directory in sys.path for locating generated code')
+ parser.add_option("--port", type="int", dest="port",
+ help="connect to server at port")
+ parser.add_option("--host", type="string", dest="host",
+ help="connect to server")
+ parser.add_option("--zlib", action="store_true", dest="zlib",
+ help="use zlib wrapper for compressed transport")
+ parser.add_option("--ssl", action="store_true", dest="ssl",
+ help="use SSL for encrypted transport")
+ parser.add_option('-v', '--verbose', action="store_const",
+ dest="verbose", const=2,
+ help="verbose output")
+ parser.add_option('-q', '--quiet', action="store_const",
+ dest="verbose", const=0,
+ help="minimal output")
+ parser.add_option('--protocol', dest="proto", type="string",
+ help="protocol to use, one of: binary, compact")
+ parser.add_option('--transport', dest="trans", type="string",
+ help="transport to use, one of: buffered, framed")
+ parser.set_defaults(framed=False, verbose=1, host='localhost', port=9090, proto='binary')
+ options, args = parser.parse_args()
+
+ if options.genpydir:
+ sys.path.insert(0, os.path.join(SCRIPT_DIR, options.genpydir))
+ sys.path.insert(0, local_libpath())
+
+ from ThriftTest import ThriftTest
+ from ThriftTest.ttypes import Xtruct, Xtruct2, Numberz, Xception, Xception2
+ from thrift.Thrift import TException
+ import thrift.TAsyncio as TAsyncio
+
+ OwnArgsTestProgram(defaultTest="suite", testRunner=unittest.TextTestRunner(verbosity=1))
diff --git a/test/py/TestClient.py b/test/py/TestClient.py
index 18ef66bb77..8570881f18 100755
--- a/test/py/TestClient.py
+++ b/test/py/TestClient.py
@@ -241,7 +241,7 @@ def testOneway(self):
start = time.time()
self.client.testOneway(1) # type is int, not float
end = time.time()
- self.assertTrue(end - start < 3,
+ self.assertTrue(end - start < 1,
"oneway sleep took %f sec" % (end - start))
def testOnewayThenNormal(self):
diff --git a/test/py/generate.cmake b/test/py/generate.cmake
index 44c53571f4..f704626d01 100644
--- a/test/py/generate.cmake
+++ b/test/py/generate.cmake
@@ -13,6 +13,7 @@ generate(${MY_PROJECT_DIR}/test/ThriftTest.thrift py:old_style gen-py-oldstyle)
generate(${MY_PROJECT_DIR}/test/ThriftTest.thrift py:no_utf8strings gen-py-no_utf8strings)
generate(${MY_PROJECT_DIR}/test/ThriftTest.thrift py:dynamic gen-py-dynamic)
generate(${MY_PROJECT_DIR}/test/ThriftTest.thrift py:dynamic,slots gen-py-dynamicslots)
+generate(${MY_PROJECT_DIR}/test/ThriftTest.thrift py:asyncio gen-py-asyncio)
generate(${MY_PROJECT_DIR}/test/DebugProtoTest.thrift py gen-py-default)
generate(${MY_PROJECT_DIR}/test/DebugProtoTest.thrift py:slots gen-py-slots)
@@ -20,3 +21,4 @@ generate(${MY_PROJECT_DIR}/test/DebugProtoTest.thrift py:old_style gen-py-oldsty
generate(${MY_PROJECT_DIR}/test/DebugProtoTest.thrift py:no_utf8strings gen-py-no_utf8strings)
generate(${MY_PROJECT_DIR}/test/DebugProtoTest.thrift py:dynamic gen-py-dynamic)
generate(${MY_PROJECT_DIR}/test/DebugProtoTest.thrift py:dynamic,slots gen-py-dynamicslots)
+generate(${MY_PROJECT_DIR}/test/DebugProtoTest.thrift py:asyncio gen-py-asyncio)
diff --git a/test/tests.json b/test/tests.json
index 3938c578df..da727137f5 100644
--- a/test/tests.json
+++ b/test/tests.json
@@ -267,6 +267,32 @@
],
"workdir": "py"
},
+ {
+ "comment": "Using 'python3' executable with asyncio",
+ "name": "py3.asyncio",
+ "client": {
+ "timeout": 10,
+ "command": [
+ "python3",
+ "TestAsyncioClient.py",
+ "--host=localhost",
+ "--genpydir=gen-py-asyncio"
+ ]
+ },
+ "transports": [
+ "buffered",
+ "framed"
+ ],
+ "sockets": [
+ "ip-ssl",
+ "ip"
+ ],
+ "protocols": [
+ "compact",
+ "binary"
+ ],
+ "workdir": "py"
+ },
{
"name": "cpp",
"server": {
diff --git a/tutorial/Makefile.am b/tutorial/Makefile.am
index 5865c54aa8..3ffac779a8 100755
--- a/tutorial/Makefile.am
+++ b/tutorial/Makefile.am
@@ -44,6 +44,9 @@ if WITH_PYTHON
SUBDIRS += py
SUBDIRS += py.twisted
SUBDIRS += py.tornado
+if WITH_PY3
+SUBDIRS += py.asyncio
+endif
endif
if WITH_RUBY
diff --git a/tutorial/py.asyncio/Makefile.am b/tutorial/py.asyncio/Makefile.am
new file mode 100755
index 0000000000..276baaf1c4
--- /dev/null
+++ b/tutorial/py.asyncio/Makefile.am
@@ -0,0 +1,39 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+THRIFT = $(top_builddir)/compiler/cpp/thrift
+
+gen-py.asyncio/tutorial/Calculator.py gen-py.asyncio/shared/SharedService.py: $(top_srcdir)/tutorial/tutorial.thrift
+ $(THRIFT) --gen py:asyncio -r $<
+
+all-local: gen-py.asyncio/tutorial/Calculator.py
+
+tutorialserver: all
+ ${PYTHON} PythonServer.py
+
+tutorialclient: all
+ ${PYTHON} PythonClient.py
+
+clean-local:
+ $(RM) -r gen-*
+
+EXTRA_DIST = \
+ setup.cfg \
+ PythonServer.py \
+ PythonClient.py
diff --git a/tutorial/py.asyncio/PythonClient.py b/tutorial/py.asyncio/PythonClient.py
new file mode 100755
index 0000000000..60cf0030da
--- /dev/null
+++ b/tutorial/py.asyncio/PythonClient.py
@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import asyncio
+import sys
+import glob
+sys.path.append('gen-py.asyncio')
+sys.path.insert(0, glob.glob('../../lib/py/build/lib*')[0])
+
+from tutorial import Calculator
+from tutorial.ttypes import InvalidOperation, Operation, Work
+
+from thrift import Thrift
+from thrift.TAsyncio import TAsyncioBufferedTransport, TAsyncioBinaryProtocol
+
+
+@asyncio.coroutine
+def main():
+ # Initialize transport with buffered asyncio reader and writer
+ transport = yield from TAsyncioBufferedTransport.connect('localhost', 9090)
+
+ # Wrap in a protocol
+ protocol = TAsyncioBinaryProtocol(transport)
+
+ # Create a client to use the protocol encoder
+ client = Calculator.Client(protocol)
+
+ yield from client.ping()
+ print('ping()')
+
+ sum_ = yield from client.add(1, 1)
+ print('1+1=%d' % sum_)
+
+ work = Work()
+
+ work.op = Operation.DIVIDE
+ work.num1 = 1
+ work.num2 = 0
+
+ try:
+ quotient = yield from client.calculate(1, work)
+ print('Whoa? You know how to divide by zero?')
+ print('FYI the answer is %d' % quotient)
+ except InvalidOperation as e:
+ print('InvalidOperation: %r' % e)
+
+ work.op = Operation.SUBTRACT
+ work.num1 = 15
+ work.num2 = 10
+
+ diff = yield from client.calculate(1, work)
+ print('15-10=%d' % diff)
+
+ log = yield from client.getStruct(1)
+ print('Check log: %s' % log.value)
+
+ # Close! - sync op
+ transport.close()
+
+if __name__ == '__main__':
+ try:
+ asyncio.get_event_loop().run_until_complete(main())
+ except Thrift.TException as tx:
+ print('%s' % tx.message)
diff --git a/tutorial/py.asyncio/PythonServer.py b/tutorial/py.asyncio/PythonServer.py
new file mode 100755
index 0000000000..6fe1b4e169
--- /dev/null
+++ b/tutorial/py.asyncio/PythonServer.py
@@ -0,0 +1,104 @@
+#!/usr/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Server is not yet ported to asyncio
+
+import glob
+import sys
+sys.path.append('../py/gen-py')
+sys.path.insert(0, glob.glob('../../lib/py/build/lib*')[0])
+
+from tutorial import Calculator
+from tutorial.ttypes import InvalidOperation, Operation
+
+from shared.ttypes import SharedStruct
+
+from thrift.transport import TSocket
+from thrift.transport import TTransport
+from thrift.protocol import TBinaryProtocol
+from thrift.server import TServer
+
+
+class CalculatorHandler:
+ def __init__(self):
+ self.log = {}
+
+ def ping(self):
+ print('ping()')
+
+ def add(self, n1, n2):
+ print('add(%d,%d)' % (n1, n2))
+ return n1 + n2
+
+ def calculate(self, logid, work):
+ print('calculate(%d, %r)' % (logid, work))
+
+ if work.op == Operation.ADD:
+ val = work.num1 + work.num2
+ elif work.op == Operation.SUBTRACT:
+ val = work.num1 - work.num2
+ elif work.op == Operation.MULTIPLY:
+ val = work.num1 * work.num2
+ elif work.op == Operation.DIVIDE:
+ if work.num2 == 0:
+ x = InvalidOperation()
+ x.whatOp = work.op
+ x.why = 'Cannot divide by 0'
+ raise x
+ val = work.num1 / work.num2
+ else:
+ x = InvalidOperation()
+ x.whatOp = work.op
+ x.why = 'Invalid operation'
+ raise x
+
+ log = SharedStruct()
+ log.key = logid
+ log.value = '%d' % (val)
+ self.log[logid] = log
+
+ return val
+
+ def getStruct(self, key):
+ print('getStruct(%d)' % (key))
+ return self.log[key]
+
+ def zip(self):
+ print('zip()')
+
+if __name__ == '__main__':
+ handler = CalculatorHandler()
+ processor = Calculator.Processor(handler)
+ transport = TSocket.TServerSocket(port=9090)
+ tfactory = TTransport.TBufferedTransportFactory()
+ pfactory = TBinaryProtocol.TBinaryProtocolFactory()
+
+ server = TServer.TSimpleServer(processor, transport, tfactory, pfactory)
+
+ # You could do one of these for a multithreaded server
+ # server = TServer.TThreadedServer(
+ # processor, transport, tfactory, pfactory)
+ # server = TServer.TThreadPoolServer(
+ # processor, transport, tfactory, pfactory)
+
+ print('Starting the server...')
+ server.serve()
+ print('done.')
diff --git a/tutorial/py.asyncio/setup.cfg b/tutorial/py.asyncio/setup.cfg
new file mode 100644
index 0000000000..2a7120a29b
--- /dev/null
+++ b/tutorial/py.asyncio/setup.cfg
@@ -0,0 +1,2 @@
+[flake8]
+ignore = E402
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
> Support Python 3.4+ asyncio support
> -----------------------------------
>
> Key: THRIFT-3770
> URL: https://issues.apache.org/jira/browse/THRIFT-3770
> Project: Thrift
> Issue Type: Bug
> Components: Python - Compiler, Python - Library
> Affects Versions: 1.0
> Reporter: Vadim Markovtsev
> Priority: Minor
> Labels: features
>
> Currently, Tornado and Twisted async engines are supported in client lib and the compiler. asyncio is a relatively new engine which is included into Python 3.4+ standard library. It is gaining popularity fast.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)