You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@thrift.apache.org by ns...@apache.org on 2016/01/04 19:03:23 UTC
[1/3] thrift git commit: THRIFT-3516 Add feature test for THeader
TBinaryProtocol
Repository: thrift
Updated Branches:
refs/heads/master 7b69c1889 -> 378b727f8
THRIFT-3516 Add feature test for THeader TBinaryProtocol
This closes #767
Project: http://git-wip-us.apache.org/repos/asf/thrift/repo
Commit: http://git-wip-us.apache.org/repos/asf/thrift/commit/378b727f
Tree: http://git-wip-us.apache.org/repos/asf/thrift/tree/378b727f
Diff: http://git-wip-us.apache.org/repos/asf/thrift/diff/378b727f
Branch: refs/heads/master
Commit: 378b727f8ec6a7a218b9b3d63cc1b0ffdf251826
Parents: 33744b0
Author: Nobuaki Sukegawa <ns...@apache.org>
Authored: Sun Jan 3 17:04:50 2016 +0900
Committer: Nobuaki Sukegawa <ns...@apache.org>
Committed: Tue Jan 5 03:02:35 2016 +0900
----------------------------------------------------------------------
Makefile.am | 1 +
test/crossrunner/__init__.py | 3 +-
test/crossrunner/collect.py | 22 +++-
test/crossrunner/prepare.py | 55 ----------
test/crossrunner/report.py | 10 +-
test/crossrunner/run.py | 20 ++--
test/features/local_thrift/__init__.py | 14 +++
test/features/tests.json | 27 +++++
test/features/theader_binary.py | 56 +++++++++++
test/features/util.py | 15 +++
test/test.py | 150 ++++++++++++++++++++--------
11 files changed, 255 insertions(+), 118 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/thrift/blob/378b727f/Makefile.am
----------------------------------------------------------------------
diff --git a/Makefile.am b/Makefile.am
index 8eb2d95..3eaa94e 100755
--- a/Makefile.am
+++ b/Makefile.am
@@ -53,6 +53,7 @@ CROSS_LANGS = @MAYBE_CPP@ @MAYBE_C_GLIB@ @MAYBE_JAVA@ @MAYBE_CSHARP@ @MAYBE_PYTH
CROSS_LANGS_COMMA_SEPARATED = $(subst $(space),$(comma),$(CROSS_LANGS))
cross: precross
+ $(PYTHON) test/test.py -F.* -s --server $(CROSS_LANGS_COMMA_SEPARATED)
$(PYTHON) test/test.py -s --server $(CROSS_LANGS_COMMA_SEPARATED) --client $(CROSS_LANGS_COMMA_SEPARATED)
TIMES = 1 2 3
http://git-wip-us.apache.org/repos/asf/thrift/blob/378b727f/test/crossrunner/__init__.py
----------------------------------------------------------------------
diff --git a/test/crossrunner/__init__.py b/test/crossrunner/__init__.py
index 584cc07..49025ed 100644
--- a/test/crossrunner/__init__.py
+++ b/test/crossrunner/__init__.py
@@ -18,7 +18,6 @@
#
from .test import test_name
-from .collect import collect_tests
+from .collect import collect_cross_tests, collect_feature_tests
from .run import TestDispatcher
from .report import generate_known_failures, load_known_failures
-from .prepare import prepare
http://git-wip-us.apache.org/repos/asf/thrift/blob/378b727f/test/crossrunner/collect.py
----------------------------------------------------------------------
diff --git a/test/crossrunner/collect.py b/test/crossrunner/collect.py
index c6e33e9..455189c 100644
--- a/test/crossrunner/collect.py
+++ b/test/crossrunner/collect.py
@@ -18,6 +18,7 @@
#
import platform
+import re
from itertools import product
from .util import merge_dict
@@ -51,7 +52,7 @@ DEFAULT_DELAY = 1
DEFAULT_TIMEOUT = 5
-def collect_testlibs(config, server_match, client_match):
+def _collect_testlibs(config, server_match, client_match=[None]):
"""Collects server/client configurations from library configurations"""
def expand_libs(config):
for lib in config:
@@ -73,7 +74,12 @@ def collect_testlibs(config, server_match, client_match):
return servers, clients
-def do_collect_tests(servers, clients):
+def collect_features(config, match):
+ res = list(map(re.compile, match))
+ return list(filter(lambda c: any(map(lambda r: r.search(c['name']), res)), config))
+
+
+def _do_collect_tests(servers, clients):
def intersection(key, o1, o2):
"""intersection of two collections.
collections are replaced with sets the first time"""
@@ -137,6 +143,12 @@ def do_collect_tests(servers, clients):
}
-def collect_tests(tests_dict, server_match, client_match):
- sv, cl = collect_testlibs(tests_dict, server_match, client_match)
- return list(do_collect_tests(sv, cl))
+def collect_cross_tests(tests_dict, server_match, client_match):
+ sv, cl = _collect_testlibs(tests_dict, server_match, client_match)
+ return list(_do_collect_tests(sv, cl))
+
+
+def collect_feature_tests(tests_dict, features_dict, server_match, feature_match):
+ sv, _ = _collect_testlibs(tests_dict, server_match)
+ ft = collect_features(features_dict, feature_match)
+ return list(_do_collect_tests(sv, ft))
http://git-wip-us.apache.org/repos/asf/thrift/blob/378b727f/test/crossrunner/prepare.py
----------------------------------------------------------------------
diff --git a/test/crossrunner/prepare.py b/test/crossrunner/prepare.py
deleted file mode 100644
index c6784af..0000000
--- a/test/crossrunner/prepare.py
+++ /dev/null
@@ -1,55 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-import os
-import subprocess
-
-from .collect import collect_testlibs
-
-
-def prepare(config_dict, testdir, server_match, client_match):
- libs, libs2 = collect_testlibs(config_dict, server_match, client_match)
- libs.extend(libs2)
-
- def prepares():
- for lib in libs:
- pre = lib.get('prepare')
- if pre:
- yield pre, lib['workdir']
-
- def files():
- for lib in libs:
- workdir = os.path.join(testdir, lib['workdir'])
- for c in lib['command']:
- if not c.startswith('-'):
- p = os.path.join(workdir, c)
- if not os.path.exists(p):
- yield os.path.split(p)
-
- def make(p):
- d, f = p
- with open(os.devnull, 'w') as devnull:
- return subprocess.Popen(['make', f], cwd=d, stderr=devnull)
-
- for pre, d in prepares():
- subprocess.Popen(pre, cwd=d).wait()
-
- for p in list(map(make, set(files()))):
- p.wait()
- return True
http://git-wip-us.apache.org/repos/asf/thrift/blob/378b727f/test/crossrunner/report.py
----------------------------------------------------------------------
diff --git a/test/crossrunner/report.py b/test/crossrunner/report.py
index 6d843d9..7840724 100644
--- a/test/crossrunner/report.py
+++ b/test/crossrunner/report.py
@@ -45,7 +45,7 @@ def generate_known_failures(testdir, overwrite, save, out):
if not r[success_index]:
yield TestEntry.get_name(*r)
try:
- with logfile_open(os.path.join(testdir, RESULT_JSON), 'r') as fp:
+ with logfile_open(path_join(testdir, RESULT_JSON), 'r') as fp:
results = json.load(fp)
except IOError:
sys.stderr.write('Unable to load last result. Did you run tests ?\n')
@@ -68,7 +68,7 @@ def generate_known_failures(testdir, overwrite, save, out):
def load_known_failures(testdir):
try:
- with logfile_open(os.path.join(testdir, FAIL_JSON % platform.system()), 'r') as fp:
+ with logfile_open(path_join(testdir, FAIL_JSON % platform.system()), 'r') as fp:
return json.load(fp)
except IOError:
return []
@@ -85,7 +85,7 @@ class TestReporter(object):
@classmethod
def test_logfile(cls, test_name, prog_kind, dir=None):
- relpath = os.path.join('log', '%s_%s.log' % (test_name, prog_kind))
+ relpath = path_join('log', '%s_%s.log' % (test_name, prog_kind))
return relpath if not dir else os.path.realpath(path_join(dir, relpath))
def _start(self):
@@ -207,8 +207,8 @@ class SummaryReporter(TestReporter):
def __init__(self, testdir, concurrent=True):
super(SummaryReporter, self).__init__()
self.testdir = testdir
- self.logdir = os.path.join(testdir, LOG_DIR)
- self.out_path = os.path.join(testdir, RESULT_JSON)
+ self.logdir = path_join(testdir, LOG_DIR)
+ self.out_path = path_join(testdir, RESULT_JSON)
self.concurrent = concurrent
self.out = sys.stdout
self._platform = platform.system()
http://git-wip-us.apache.org/repos/asf/thrift/blob/378b727f/test/crossrunner/run.py
----------------------------------------------------------------------
diff --git a/test/crossrunner/run.py b/test/crossrunner/run.py
index acba335..abbd70b 100644
--- a/test/crossrunner/run.py
+++ b/test/crossrunner/run.py
@@ -110,13 +110,13 @@ class ExecutionContext(object):
return self.proc.returncode if self.proc else None
-def exec_context(port, testdir, test, prog):
- report = ExecReporter(testdir, test, prog)
+def exec_context(port, logdir, test, prog):
+ report = ExecReporter(logdir, test, prog)
prog.build_command(port)
return ExecutionContext(prog.command, prog.workdir, prog.env, report)
-def run_test(testdir, test_dict, async=True, max_retry=3):
+def run_test(testdir, logdir, test_dict, async=True, max_retry=3):
try:
logger = multiprocessing.get_logger()
retry_count = 0
@@ -128,8 +128,8 @@ def run_test(testdir, test_dict, async=True, max_retry=3):
logger.debug('Start')
with PortAllocator.alloc_port_scoped(ports, test.socket) as port:
logger.debug('Start with port %d' % port)
- sv = exec_context(port, testdir, test, test.server)
- cl = exec_context(port, testdir, test, test.client)
+ sv = exec_context(port, logdir, test, test.server)
+ cl = exec_context(port, logdir, test, test.client)
logger.debug('Starting server')
with sv.start():
@@ -256,9 +256,10 @@ class NonAsyncResult(object):
class TestDispatcher(object):
- def __init__(self, testdir, concurrency):
+ def __init__(self, testdir, logdir, concurrency):
self._log = multiprocessing.get_logger()
self.testdir = testdir
+ self.logdir = logdir
# seems needed for python 2.x to handle keyboard interrupt
self._stop = multiprocessing.Event()
self._async = concurrency > 1
@@ -273,7 +274,7 @@ class TestDispatcher(object):
self._m.register('ports', PortAllocator)
self._m.start()
self._pool = multiprocessing.Pool(concurrency, self._pool_init, (self._m.address,))
- self._report = SummaryReporter(testdir, concurrency > 1)
+ self._report = SummaryReporter(logdir, concurrency > 1)
self._log.debug(
'TestDispatcher started with %d concurrent jobs' % concurrency)
@@ -287,12 +288,13 @@ class TestDispatcher(object):
ports = m.ports()
def _dispatch_sync(self, test, cont):
- r = run_test(self.testdir, test, False)
+ r = run_test(self.testdir, self.logdir, test, False)
cont(r)
return NonAsyncResult(r)
def _dispatch_async(self, test, cont):
- return self._pool.apply_async(func=run_test, args=(self.testdir, test,), callback=cont)
+ self._log.debug('_dispatch_async')
+ return self._pool.apply_async(func=run_test, args=(self.testdir, self.logdir, test,), callback=cont)
def dispatch(self, test):
index = self._report.add_test(test)
http://git-wip-us.apache.org/repos/asf/thrift/blob/378b727f/test/features/local_thrift/__init__.py
----------------------------------------------------------------------
diff --git a/test/features/local_thrift/__init__.py b/test/features/local_thrift/__init__.py
new file mode 100644
index 0000000..383ee5f
--- /dev/null
+++ b/test/features/local_thrift/__init__.py
@@ -0,0 +1,14 @@
+import os
+import sys
+
+SCRIPT_DIR = os.path.realpath(os.path.dirname(__file__))
+ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(SCRIPT_DIR)))
+
+if sys.version_info[0] == 2:
+ import glob
+ libdir = glob.glob(os.path.join(ROOT_DIR, 'lib', 'py', 'build', 'lib.*'))[0]
+ sys.path.insert(0, libdir)
+ thrift = __import__('thrift')
+else:
+ sys.path.insert(0, os.path.join(ROOT_DIR, 'lib', 'py', 'build', 'lib'))
+ thrift = __import__('thrift')
http://git-wip-us.apache.org/repos/asf/thrift/blob/378b727f/test/features/tests.json
----------------------------------------------------------------------
diff --git a/test/features/tests.json b/test/features/tests.json
new file mode 100644
index 0000000..0836309
--- /dev/null
+++ b/test/features/tests.json
@@ -0,0 +1,27 @@
+[
+ {
+ "description": "THeader detects unframed binary wire format",
+ "name": "theader_unframed_binary",
+ "command": [
+ "python",
+ "theader_binary.py"
+ ],
+ "protocols": ["header"],
+ "transports": ["buffered"],
+ "sockets": ["ip"],
+ "workdir": "features"
+ },
+ {
+ "description": "THeader detects framed binary wire format",
+ "name": "theader_framed_binary",
+ "command": [
+ "python",
+ "theader_binary.py",
+ "--override-transport=framed"
+ ],
+ "protocols": ["header"],
+ "transports": ["buffered"],
+ "sockets": ["ip"],
+ "workdir": "features"
+ }
+]
http://git-wip-us.apache.org/repos/asf/thrift/blob/378b727f/test/features/theader_binary.py
----------------------------------------------------------------------
diff --git a/test/features/theader_binary.py b/test/features/theader_binary.py
new file mode 100644
index 0000000..0316741
--- /dev/null
+++ b/test/features/theader_binary.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+
+import argparse
+import socket
+import sys
+
+from util import add_common_args
+from local_thrift import thrift
+from thrift.Thrift import TMessageType, TType
+from thrift.transport.TSocket import TSocket
+from thrift.transport.TTransport import TBufferedTransport, TFramedTransport
+from thrift.protocol.TBinaryProtocol import TBinaryProtocol
+
+
+# THeader stack should accept binary protocol with optionally framed transport
+def main(argv):
+ p = argparse.ArgumentParser()
+ add_common_args(p)
+ # Since THeaderTransport acts as framed transport when detected frame, we
+ # cannot use --transport=framed as it would result in 2 layered frames.
+ p.add_argument('--override-transport')
+ args = p.parse_args()
+ assert args.protocol == 'header'
+ assert args.transport == 'buffered'
+ assert not args.ssl
+
+ sock = TSocket(args.host, args.port, socket_family=socket.AF_INET)
+ if not args.override_transport or args.override_transport == 'buffered':
+ trans = TBufferedTransport(sock)
+ elif args.override_transport == 'framed':
+ trans = TFramedTransport(sock)
+ else:
+ raise ValueError('invalid transport')
+ trans.open()
+ proto = TBinaryProtocol(trans)
+ proto.writeMessageBegin('testVoid', TMessageType.CALL, 3)
+ proto.writeStructBegin('testVoid_args')
+ proto.writeFieldStop()
+ proto.writeStructEnd()
+ proto.writeMessageEnd()
+ trans.flush()
+
+ _, mtype, _ = proto.readMessageBegin()
+ assert mtype == TMessageType.REPLY
+ proto.readStructBegin()
+ _, ftype, _ = proto.readFieldBegin()
+ assert ftype == TType.STOP
+ proto.readFieldEnd()
+ proto.readStructEnd()
+ proto.readMessageEnd()
+
+ trans.close()
+
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv[1:]))
http://git-wip-us.apache.org/repos/asf/thrift/blob/378b727f/test/features/util.py
----------------------------------------------------------------------
diff --git a/test/features/util.py b/test/features/util.py
new file mode 100644
index 0000000..cff7ff8
--- /dev/null
+++ b/test/features/util.py
@@ -0,0 +1,15 @@
+import argparse
+
+
+def add_common_args(p):
+ p.add_argument('--host', default='localhost')
+ p.add_argument('--port', type=int)
+ p.add_argument('--protocol')
+ p.add_argument('--transport')
+ p.add_argument('--ssl', action='store_true')
+
+
+def parse_common_args(argv):
+ p = argparse.ArgumentParser()
+ add_common_args(p)
+ return p.parse_args(argv)
http://git-wip-us.apache.org/repos/asf/thrift/blob/378b727f/test/test.py
----------------------------------------------------------------------
diff --git a/test/test.py b/test/test.py
index 1176369..0c799b9 100755
--- a/test/test.py
+++ b/test/test.py
@@ -25,39 +25,102 @@
# This script supports python 2.7 and later.
# python 3.x is recommended for better stability.
#
-# TODO: eliminate a few 2.7 occurrences to support 2.6 ?
-#
+from __future__ import print_function
+from itertools import chain
import json
import logging
import multiprocessing
-import optparse
+import argparse
import os
import sys
import crossrunner
+from crossrunner.compat import path_join
TEST_DIR = os.path.realpath(os.path.dirname(__file__))
-CONFIG_PATH = os.path.join(TEST_DIR, 'tests.json')
+CONFIG_FILE = 'tests.json'
-def prepare(server_match, client_match):
- with open(CONFIG_PATH, 'r') as fp:
+def run_tests(collect_func, basedir, server_match, client_match, jobs, skip):
+ logger = multiprocessing.get_logger()
+ logger.debug('Collecting tests')
+ with open(path_join(basedir, CONFIG_FILE), 'r') as fp:
j = json.load(fp)
- return crossrunner.prepare(j, TEST_DIR, server_match, client_match)
+ tests = collect_func(j, server_match, client_match)
+ if not tests:
+ print('No test found that matches the criteria', file=sys.stderr)
+ # print(' servers: %s' % server_match, file=sys.stderr)
+ # print(' clients: %s' % client_match, file=sys.stderr)
+ return False
+ if skip:
+ logger.debug('Skipping known failures')
+ known = crossrunner.load_known_failures(basedir)
+ tests = list(filter(lambda t: crossrunner.test_name(**t) not in known, tests))
+
+ dispatcher = crossrunner.TestDispatcher(TEST_DIR, basedir, jobs)
+ logger.debug('Executing %d tests' % len(tests))
+ try:
+ for r in [dispatcher.dispatch(test) for test in tests]:
+ r.wait()
+ logger.debug('Waiting for completion')
+ return dispatcher.wait()
+ except (KeyboardInterrupt, SystemExit):
+ logger.debug('Interrupted, shutting down')
+ dispatcher.terminate()
+ return False
-def run_tests(server_match, client_match, jobs, skip_known_failures):
+def run_cross_tests(server_match, client_match, jobs, skip_known_failures):
logger = multiprocessing.get_logger()
logger.debug('Collecting tests')
- with open(CONFIG_PATH, 'r') as fp:
+ with open(path_join(TEST_DIR, CONFIG_FILE), 'r') as fp:
j = json.load(fp)
- tests = list(crossrunner.collect_tests(j, server_match, client_match))
+ tests = crossrunner.collect_cross_tests(j, server_match, client_match)
+ if not tests:
+ print('No test found that matches the criteria', file=sys.stderr)
+ print(' servers: %s' % server_match, file=sys.stderr)
+ print(' clients: %s' % client_match, file=sys.stderr)
+ return False
if skip_known_failures:
+ logger.debug('Skipping known failures')
known = crossrunner.load_known_failures(TEST_DIR)
tests = list(filter(lambda t: crossrunner.test_name(**t) not in known, tests))
- dispatcher = crossrunner.TestDispatcher(TEST_DIR, jobs)
+ dispatcher = crossrunner.TestDispatcher(TEST_DIR, TEST_DIR, jobs)
+ logger.debug('Executing %d tests' % len(tests))
+ try:
+ for r in [dispatcher.dispatch(test) for test in tests]:
+ r.wait()
+ logger.debug('Waiting for completion')
+ return dispatcher.wait()
+ except (KeyboardInterrupt, SystemExit):
+ logger.debug('Interrupted, shutting down')
+ dispatcher.terminate()
+ return False
+
+
+def run_feature_tests(server_match, feature_match, jobs, skip_known_failures):
+ basedir = path_join(TEST_DIR, 'features')
+ # run_tests(crossrunner.collect_feature_tests, basedir, server_match, feature_match, jobs, skip_known_failures)
+ logger = multiprocessing.get_logger()
+ logger.debug('Collecting tests')
+ with open(path_join(TEST_DIR, CONFIG_FILE), 'r') as fp:
+ j = json.load(fp)
+ with open(path_join(basedir, CONFIG_FILE), 'r') as fp:
+ j2 = json.load(fp)
+ tests = crossrunner.collect_feature_tests(j, j2, server_match, feature_match)
+ if not tests:
+ print('No test found that matches the criteria', file=sys.stderr)
+ print(' servers: %s' % server_match, file=sys.stderr)
+ print(' features: %s' % feature_match, file=sys.stderr)
+ return False
+ if skip_known_failures:
+ logger.debug('Skipping known failures')
+ known = crossrunner.load_known_failures(basedir)
+ tests = list(filter(lambda t: crossrunner.test_name(**t) not in known, tests))
+
+ dispatcher = crossrunner.TestDispatcher(TEST_DIR, basedir, jobs)
logger.debug('Executing %d tests' % len(tests))
try:
for r in [dispatcher.dispatch(test) for test in tests]:
@@ -79,44 +142,47 @@ def default_concurrenty():
def main(argv):
- parser = optparse.OptionParser()
- parser.add_option('--server', type='string', dest='servers', default='',
- help='list of servers to test separated by commas, eg:- --server=cpp,java')
- parser.add_option('--client', type='string', dest='clients', default='',
- help='list of clients to test separated by commas, eg:- --client=cpp,java')
- parser.add_option('-s', '--skip-known-failures', action='store_true', dest='skip_known_failures',
- help='do not execute tests that are known to fail')
- parser.add_option('-j', '--jobs', type='int', dest='jobs',
- default=default_concurrenty(),
- help='number of concurrent test executions')
- g = optparse.OptionGroup(parser, 'Advanced')
- g.add_option('-v', '--verbose', action='store_const',
- dest='log_level', const=logging.DEBUG, default=logging.WARNING,
- help='show debug output for test runner')
- g.add_option('-P', '--print-expected-failures', choices=['merge', 'overwrite'],
- dest='print_failures', default=None,
- help="generate expected failures based on last result and print to stdout")
- g.add_option('-U', '--update-expected-failures', choices=['merge', 'overwrite'],
- dest='update_failures', default=None,
- help="generate expected failures based on last result and save to default file location")
- g.add_option('--prepare', action='store_true',
- dest='prepare',
- help="try to prepare files needed for cross test (experimental)")
- parser.add_option_group(g)
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--server', default='', nargs='*',
+ help='list of servers to test')
+ parser.add_argument('--client', default='', nargs='*',
+ help='list of clients to test')
+ parser.add_argument('-s', '--skip-known-failures', action='store_true', dest='skip_known_failures',
+ help='do not execute tests that are known to fail')
+ parser.add_argument('-j', '--jobs', type=int,
+ default=default_concurrenty(),
+ help='number of concurrent test executions')
+ parser.add_argument('-F', '--features', nargs='*', default=None,
+ help='run feature tests instead of cross language tests')
+
+ g = parser.add_argument_group(title='Advanced')
+ g.add_argument('-v', '--verbose', action='store_const',
+ dest='log_level', const=logging.DEBUG, default=logging.WARNING,
+ help='show debug output for test runner')
+ g.add_argument('-P', '--print-expected-failures', choices=['merge', 'overwrite'],
+ dest='print_failures',
+ help="generate expected failures based on last result and print to stdout")
+ g.add_argument('-U', '--update-expected-failures', choices=['merge', 'overwrite'],
+ dest='update_failures',
+ help="generate expected failures based on last result and save to default file location")
+ options = parser.parse_args(argv)
+
logger = multiprocessing.log_to_stderr()
- options, _ = parser.parse_args(argv)
- server_match = options.servers.split(',') if options.servers else []
- client_match = options.clients.split(',') if options.clients else []
logger.setLevel(options.log_level)
- if options.prepare:
- res = prepare(server_match, client_match)
- elif options.update_failures or options.print_failures:
+ # Allow multiple args separated with ',' for backward compatibility
+ server_match = list(chain(*[x.split(',') for x in options.server]))
+ client_match = list(chain(*[x.split(',') for x in options.client]))
+
+ if options.update_failures or options.print_failures:
res = crossrunner.generate_known_failures(
TEST_DIR, options.update_failures == 'overwrite',
options.update_failures, options.print_failures)
+ elif options.features is not None:
+ features = options.features or ['.*']
+ res = run_feature_tests(server_match, features, options.jobs, options.skip_known_failures)
else:
- res = run_tests(server_match, client_match, options.jobs, options.skip_known_failures)
+ res = run_cross_tests(server_match, client_match, options.jobs, options.skip_known_failures)
return 0 if res else 1
if __name__ == '__main__':
[2/3] thrift git commit: THRIFT-3515 Python 2.6 compatibility and
test on CI
Posted by ns...@apache.org.
THRIFT-3515 Python 2.6 compatibility and test on CI
This closes #766
Project: http://git-wip-us.apache.org/repos/asf/thrift/repo
Commit: http://git-wip-us.apache.org/repos/asf/thrift/commit/33744b05
Tree: http://git-wip-us.apache.org/repos/asf/thrift/tree/33744b05
Diff: http://git-wip-us.apache.org/repos/asf/thrift/diff/33744b05
Branch: refs/heads/master
Commit: 33744b0524b7248dda9e9e544420d69c33d3a3aa
Parents: 1d8e745
Author: Nobuaki Sukegawa <ns...@apache.org>
Authored: Sun Jan 3 14:24:39 2016 +0900
Committer: Nobuaki Sukegawa <ns...@apache.org>
Committed: Tue Jan 5 03:02:35 2016 +0900
----------------------------------------------------------------------
.travis.yml | 6 +++
build/cmake/DefineOptions.cmake | 70 +++++++++++++++-------------
build/cmake/DefinePlatformSpecifc.cmake | 0
build/docker/centos6/Dockerfile | 49 +++++++++++++++++++
build/docker/centos6/scripts/keepit | 1 +
configure.ac | 2 +-
lib/py/src/compat.py | 2 +-
lib/py/src/protocol/TCompactProtocol.py | 4 +-
lib/py/src/protocol/TJSONProtocol.py | 4 +-
test/py/SerializationTest.py | 4 ++
10 files changed, 103 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/thrift/blob/33744b05/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 97440f3..88e9745 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -113,6 +113,12 @@ env:
BUILD_ENV="-e CC=gcc -e CXX=g++"
DISTRO=centos
+ - TEST_NAME="Python 2.6 (CentOS 6)"
+ BUILD_CMD="../cmake.sh"
+ BUILD_ARG="-DWITH_PYTHON=ON -DWITH_CPP=OFF -DWITH_JAVA=OFF -DWITH_HASKELL=OFF"
+ BUILD_ENV="-e CC=gcc -e CXX=g++"
+ DISTRO=centos6
+
# Distribution
- TEST_NAME="make dist"
BUILD_CMD="../make-dist.sh"
http://git-wip-us.apache.org/repos/asf/thrift/blob/33744b05/build/cmake/DefineOptions.cmake
----------------------------------------------------------------------
diff --git a/build/cmake/DefineOptions.cmake b/build/cmake/DefineOptions.cmake
index 342e040..695a615 100644
--- a/build/cmake/DefineOptions.cmake
+++ b/build/cmake/DefineOptions.cmake
@@ -42,46 +42,50 @@ option(BUILD_LIBRARIES "Build Thrift libraries" ON)
# C++
option(WITH_CPP "Build C++ Thrift library" ON)
-find_package(Boost 1.53 QUIET)
+if(WITH_CPP)
+ find_package(Boost 1.53 QUIET)
+ # NOTE: Currently the following options are C++ specific,
+ # but in future other libraries might reuse them.
+ # So they are not dependent on WITH_CPP but setting them without WITH_CPP currently
+ # has no effect.
+ if(ZLIB_LIBRARY)
+ # FindZLIB.cmake does not normalize path so we need to do it ourselves.
+ file(TO_CMAKE_PATH ${ZLIB_LIBRARY} ZLIB_LIBRARY)
+ endif()
+ find_package(ZLIB QUIET)
+ CMAKE_DEPENDENT_OPTION(WITH_ZLIB "Build with ZLIB support" ON
+ "ZLIB_FOUND" OFF)
+ find_package(Libevent QUIET)
+ CMAKE_DEPENDENT_OPTION(WITH_LIBEVENT "Build with libevent support" ON
+ "Libevent_FOUND" OFF)
+ find_package(Qt4 QUIET COMPONENTS QtCore QtNetwork)
+ CMAKE_DEPENDENT_OPTION(WITH_QT4 "Build with Qt4 support" ON
+ "QT4_FOUND" OFF)
+ find_package(Qt5 QUIET COMPONENTS Core Network)
+ CMAKE_DEPENDENT_OPTION(WITH_QT5 "Build with Qt5 support" ON
+ "Qt5_FOUND" OFF)
+ if(${WITH_QT4} AND ${WITH_QT5} AND ${CMAKE_MAJOR_VERSION} LESS 3)
+ # cmake < 3.0.0 causes conflict when building both Qt4 and Qt5
+ set(WITH_QT4 OFF)
+ endif()
+ find_package(OpenSSL QUIET)
+ CMAKE_DEPENDENT_OPTION(WITH_OPENSSL "Build with OpenSSL support" ON
+ "OPENSSL_FOUND" OFF)
+ option(WITH_STDTHREADS "Build with C++ std::thread support" OFF)
+ CMAKE_DEPENDENT_OPTION(WITH_BOOSTTHREADS "Build with Boost threads support" OFF
+ "NOT WITH_STDTHREADS;Boost_FOUND" OFF)
+endif()
CMAKE_DEPENDENT_OPTION(BUILD_CPP "Build C++ library" ON
"BUILD_LIBRARIES;WITH_CPP;Boost_FOUND" OFF)
-# NOTE: Currently the following options are C++ specific,
-# but in future other libraries might reuse them.
-# So they are not dependent on WITH_CPP but setting them without WITH_CPP currently
-# has no effect.
-if(ZLIB_LIBRARY)
- # FindZLIB.cmake does not normalize path so we need to do it ourselves.
- file(TO_CMAKE_PATH ${ZLIB_LIBRARY} ZLIB_LIBRARY)
-endif()
-find_package(ZLIB QUIET)
-CMAKE_DEPENDENT_OPTION(WITH_ZLIB "Build with ZLIB support" ON
- "ZLIB_FOUND" OFF)
-find_package(Libevent QUIET)
-CMAKE_DEPENDENT_OPTION(WITH_LIBEVENT "Build with libevent support" ON
- "Libevent_FOUND" OFF)
-find_package(Qt4 QUIET COMPONENTS QtCore QtNetwork)
-CMAKE_DEPENDENT_OPTION(WITH_QT4 "Build with Qt4 support" ON
- "QT4_FOUND" OFF)
-find_package(Qt5 QUIET COMPONENTS Core Network)
-CMAKE_DEPENDENT_OPTION(WITH_QT5 "Build with Qt5 support" ON
- "Qt5_FOUND" OFF)
-if(${WITH_QT4} AND ${WITH_QT5} AND ${CMAKE_MAJOR_VERSION} LESS 3)
- # cmake < 3.0.0 causes conflict when building both Qt4 and Qt5
- set(WITH_QT4 OFF)
-endif()
-find_package(OpenSSL QUIET)
-CMAKE_DEPENDENT_OPTION(WITH_OPENSSL "Build with OpenSSL support" ON
- "OPENSSL_FOUND" OFF)
-option(WITH_STDTHREADS "Build with C++ std::thread support" OFF)
-CMAKE_DEPENDENT_OPTION(WITH_BOOSTTHREADS "Build with Boost threads support" OFF
- "NOT WITH_STDTHREADS;Boost_FOUND" OFF)
-
# C GLib
option(WITH_C_GLIB "Build C (GLib) Thrift library" ON)
-find_package(GLIB QUIET COMPONENTS gobject)
+if(WITH_C_GLIB)
+ find_package(GLIB QUIET COMPONENTS gobject)
+endif()
CMAKE_DEPENDENT_OPTION(BUILD_C_GLIB "Build C (GLib) library" ON
"BUILD_LIBRARIES;WITH_C_GLIB;GLIB_FOUND" OFF)
+
# Java
option(WITH_JAVA "Build Java Thrift library" ON)
if(ANDROID)
http://git-wip-us.apache.org/repos/asf/thrift/blob/33744b05/build/cmake/DefinePlatformSpecifc.cmake
----------------------------------------------------------------------
diff --git a/build/cmake/DefinePlatformSpecifc.cmake b/build/cmake/DefinePlatformSpecifc.cmake
old mode 100755
new mode 100644
http://git-wip-us.apache.org/repos/asf/thrift/blob/33744b05/build/docker/centos6/Dockerfile
----------------------------------------------------------------------
diff --git a/build/docker/centos6/Dockerfile b/build/docker/centos6/Dockerfile
new file mode 100644
index 0000000..0e571c5
--- /dev/null
+++ b/build/docker/centos6/Dockerfile
@@ -0,0 +1,49 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Apache Thrift Docker build environment for Centos 6
+#
+# This file is intended for testing old packages that are not available for
+# latest Ubuntu LTS/Debian/CentOS. Currently, it is only used for Python 2.6.
+#
+
+FROM centos:6
+MAINTAINER Apache Thrift <de...@thrift.apache.org>
+
+RUN yum install -y \
+ autoconf \
+ bison \
+ bison-devel \
+ clang \
+ flex \
+ gcc \
+ gcc-c++ \
+ git \
+ libtool \
+ m4 \
+ make \
+ perl \
+ tar \
+ python-devel \
+ python-setuptools \
+ python-twisted-web \
+ python-six \
+ && yum clean all
+
+# CMake
+RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \
+ cd cmake-3.4.1 && ./bootstrap && make -j4 && make install
+
+ENV THRIFT_ROOT /thrift
+RUN mkdir -p $THRIFT_ROOT/src
+COPY scripts $THRIFT_ROOT
+WORKDIR $THRIFT_ROOT/src
http://git-wip-us.apache.org/repos/asf/thrift/blob/33744b05/build/docker/centos6/scripts/keepit
----------------------------------------------------------------------
diff --git a/build/docker/centos6/scripts/keepit b/build/docker/centos6/scripts/keepit
new file mode 100644
index 0000000..cb885df
--- /dev/null
+++ b/build/docker/centos6/scripts/keepit
@@ -0,0 +1 @@
+keep it
http://git-wip-us.apache.org/repos/asf/thrift/blob/33744b05/configure.ac
----------------------------------------------------------------------
diff --git a/configure.ac b/configure.ac
index 27299b4..141b542 100755
--- a/configure.ac
+++ b/configure.ac
@@ -277,7 +277,7 @@ AM_CONDITIONAL(WITH_LUA, [test "$have_lua" = "yes"])
AX_THRIFT_LIB(python, [Python], yes)
if test "$with_python" = "yes"; then
AC_PATH_PROG([TRIAL], [trial])
- AM_PATH_PYTHON(2.4,, :)
+ AM_PATH_PYTHON(2.6,, :)
if test -n "$TRIAL" && test "x$PYTHON" != "x" && test "x$PYTHON" != "x:" ; then
have_python="yes"
fi
http://git-wip-us.apache.org/repos/asf/thrift/blob/33744b05/lib/py/src/compat.py
----------------------------------------------------------------------
diff --git a/lib/py/src/compat.py b/lib/py/src/compat.py
index b2f47dc..06f672a 100644
--- a/lib/py/src/compat.py
+++ b/lib/py/src/compat.py
@@ -22,6 +22,6 @@ else:
def str_to_binary(str_val):
try:
- return bytearray(str_val, 'utf8')
+ return bytes(str_val, 'utf8')
except:
return str_val
http://git-wip-us.apache.org/repos/asf/thrift/blob/33744b05/lib/py/src/protocol/TCompactProtocol.py
----------------------------------------------------------------------
diff --git a/lib/py/src/protocol/TCompactProtocol.py b/lib/py/src/protocol/TCompactProtocol.py
index a8b025a..6023066 100644
--- a/lib/py/src/protocol/TCompactProtocol.py
+++ b/lib/py/src/protocol/TCompactProtocol.py
@@ -56,7 +56,7 @@ def fromZigZag(n):
def writeVarint(trans, n):
- out = []
+ out = bytearray()
while True:
if n & ~0x7f == 0:
out.append(n)
@@ -64,7 +64,7 @@ def writeVarint(trans, n):
else:
out.append((n & 0xff) | 0x80)
n = n >> 7
- trans.write(bytearray(out))
+ trans.write(bytes(out))
def readVarint(trans):
http://git-wip-us.apache.org/repos/asf/thrift/blob/33744b05/lib/py/src/protocol/TJSONProtocol.py
----------------------------------------------------------------------
diff --git a/lib/py/src/protocol/TJSONProtocol.py b/lib/py/src/protocol/TJSONProtocol.py
index acfce4a..3612e91 100644
--- a/lib/py/src/protocol/TJSONProtocol.py
+++ b/lib/py/src/protocol/TJSONProtocol.py
@@ -203,7 +203,7 @@ class TJSONProtocolBase(TProtocolBase):
json_str.append('"')
self.trans.write(str_to_binary(''.join(json_str)))
- def writeJSONNumber(self, number, formatter='{}'):
+ def writeJSONNumber(self, number, formatter='{0}'):
self.context.write()
jsNumber = str(formatter.format(number)).encode('ascii')
if self.context.escapeNum():
@@ -550,7 +550,7 @@ class TJSONProtocol(TJSONProtocolBase):
def writeDouble(self, dbl):
# 17 significant digits should be just enough for any double precision value.
- self.writeJSONNumber(dbl, '{:.17g}')
+ self.writeJSONNumber(dbl, '{0:.17g}')
def writeString(self, string):
self.writeJSONString(string)
http://git-wip-us.apache.org/repos/asf/thrift/blob/33744b05/test/py/SerializationTest.py
----------------------------------------------------------------------
diff --git a/test/py/SerializationTest.py b/test/py/SerializationTest.py
index 99e0393..d4755cf 100755
--- a/test/py/SerializationTest.py
+++ b/test/py/SerializationTest.py
@@ -24,6 +24,7 @@ from DebugProtoTest.ttypes import CompactProtoTestStruct, Empty
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol, TCompactProtocol, TJSONProtocol
from thrift.TSerialization import serialize, deserialize
+import sys
import unittest
@@ -258,6 +259,9 @@ class AbstractTest(unittest.TestCase):
self.assertTrue(len(rep) > 0)
def testIntegerLimits(self):
+ if (sys.version_info[0] == 2 and sys.version_info[1] <= 6):
+ print('Skipping testIntegerLimits for Python 2.6')
+ return
bad_values = [CompactProtoTestStruct(a_byte=128), CompactProtoTestStruct(a_byte=-129),
CompactProtoTestStruct(a_i16=32768), CompactProtoTestStruct(a_i16=-32769),
CompactProtoTestStruct(a_i32=2147483648), CompactProtoTestStruct(a_i32=-2147483649),
[3/3] thrift git commit: Update IDL spec
Posted by ns...@apache.org.
Update IDL spec
Add i8 to BaseType
Project: http://git-wip-us.apache.org/repos/asf/thrift/repo
Commit: http://git-wip-us.apache.org/repos/asf/thrift/commit/1d8e7450
Tree: http://git-wip-us.apache.org/repos/asf/thrift/tree/1d8e7450
Diff: http://git-wip-us.apache.org/repos/asf/thrift/diff/1d8e7450
Branch: refs/heads/master
Commit: 1d8e745035354e67c794bec9e0b5663ee7cab902
Parents: 7b69c18
Author: Nobuaki Sukegawa <ns...@apache.org>
Authored: Mon Jan 4 02:02:17 2016 +0900
Committer: Nobuaki Sukegawa <ns...@apache.org>
Committed: Tue Jan 5 03:02:35 2016 +0900
----------------------------------------------------------------------
doc/specs/idl.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/thrift/blob/1d8e7450/doc/specs/idl.md
----------------------------------------------------------------------
diff --git a/doc/specs/idl.md b/doc/specs/idl.md
index 6da4696..e8cf39f 100644
--- a/doc/specs/idl.md
+++ b/doc/specs/idl.md
@@ -150,7 +150,7 @@ N.B.: These have some internal purpose at Facebook but serve no current purpose
[25] DefinitionType ::= BaseType | ContainerType
- [26] BaseType ::= 'bool' | 'byte' | 'i16' | 'i32' | 'i64' | 'double' | 'string' | 'binary' | 'slist'
+ [26] BaseType ::= 'bool' | 'byte' | 'i8' | 'i16' | 'i32' | 'i64' | 'double' | 'string' | 'binary' | 'slist'
[27] ContainerType ::= MapType | SetType | ListType