You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@heron.apache.org by GitBox <gi...@apache.org> on 2018/09/12 18:45:37 UTC
[GitHub] huijunwu closed pull request #3024: run integration tests parallelly
huijunwu closed pull request #3024: run integration tests parallelly
URL: https://github.com/apache/incubator-heron/pull/3024
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/integration_test/src/python/test_runner/main.py b/integration_test/src/python/test_runner/main.py
index 83f50d83bd..ae921b3613 100644
--- a/integration_test/src/python/test_runner/main.py
+++ b/integration_test/src/python/test_runner/main.py
@@ -25,6 +25,7 @@
import time
import uuid
from httplib import HTTPConnection
+from threading import Lock, Thread
from ..common import status
from heron.common.src.python.utils import log
@@ -36,6 +37,9 @@
#seconds
RETRY_INTERVAL = 10
+successes = []
+failures = []
+
class FileBasedExpectedResultsHandler(object):
def __init__(self, file_path):
self.file_path = file_path
@@ -301,30 +305,59 @@ def filter_test_topologies(test_topologies, test_pattern):
sys.exit(1)
return test_topologies
-def run_tests(conf, args):
+def run_tests(conf, test_args):
''' Run the test for each topology specified in the conf file '''
- successes = []
- failures = []
+ lock = Lock()
timestamp = time.strftime('%Y%m%d%H%M%S')
- http_server_host_port = "%s:%d" % (args.http_server_hostname, args.http_server_port)
+ http_server_host_port = "%s:%d" % (test_args.http_server_hostname, test_args.http_server_port)
- if args.tests_bin_path.endswith("scala-integration-tests.jar"):
- test_topologies = filter_test_topologies(conf["scalaTopologies"], args.test_topology_pattern)
+ if test_args.tests_bin_path.endswith("scala-integration-tests.jar"):
+ test_topologies = filter_test_topologies(conf["scalaTopologies"], test_args.test_topology_pattern)
topology_classpath_prefix = conf["topologyClasspathPrefix"]
extra_topology_args = "-s http://%s/state" % http_server_host_port
- elif args.tests_bin_path.endswith("integration-tests.jar"):
- test_topologies = filter_test_topologies(conf["javaTopologies"], args.test_topology_pattern)
+ elif test_args.tests_bin_path.endswith("integration-tests.jar"):
+ test_topologies = filter_test_topologies(conf["javaTopologies"], test_args.test_topology_pattern)
topology_classpath_prefix = conf["topologyClasspathPrefix"]
extra_topology_args = "-s http://%s/state" % http_server_host_port
- elif args.tests_bin_path.endswith("heron_integ_topology.pex"):
- test_topologies = filter_test_topologies(conf["pythonTopologies"], args.test_topology_pattern)
+ elif test_args.tests_bin_path.endswith("heron_integ_topology.pex"):
+ test_topologies = filter_test_topologies(conf["pythonTopologies"], test_args.test_topology_pattern)
topology_classpath_prefix = ""
extra_topology_args = ""
else:
- raise ValueError("Unrecognized binary file type: %s" % args.tests_bin_path)
+ raise ValueError("Unrecognized binary file type: %s" % test_args.tests_bin_path)
+
+ def _run_single_test(topology_name, topology_conf, test_args, http_server_host_port, classpath,
+ update_args, topology_args):
+ global successes, failures
+ results_checker = load_result_checker(
+ topology_name, topology_conf,
+ load_expected_result_handler(topology_name, topology_conf, test_args, http_server_host_port),
+ HttpBasedActualResultsHandler(http_server_host_port, topology_name))
+
+ start_secs = int(time.time())
+ try:
+ result = run_test(topology_name, classpath, results_checker,
+ test_args, http_server_host_port, update_args, topology_args)
+ test_tuple = (topology_name, int(time.time()) - start_secs)
+ lock.acquire()
+ if isinstance(result, status.TestSuccess):
+ successes += [test_tuple]
+ elif isinstance(result, status.TestFailure):
+ failures += [test_tuple]
+ else:
+ logging.error("Unrecognized test response returned for test %s: %s",
+ topology_name, str(result))
+ failures += [test_tuple]
+ lock.release()
+ except status.TestFailure:
+ test_tuple = (topology_name, int(time.time()) - start_secs)
+ lock.acquire()
+ failures += [test_tuple]
+ lock.release()
current = 1
+ test_threads = []
for topology_conf in test_topologies:
topology_name = ("%s_%s_%s") % (timestamp, topology_conf["topologyName"], str(uuid.uuid4()))
classpath = topology_classpath_prefix + topology_conf["classPath"]
@@ -344,32 +377,20 @@ def run_tests(conf, args):
+ topology_name)
topology_args = "%s %s" % (topology_args, topology_conf["topologyArgs"])
- results_checker = load_result_checker(
- topology_name, topology_conf,
- load_expected_result_handler(topology_name, topology_conf, args, http_server_host_port),
- HttpBasedActualResultsHandler(http_server_host_port, topology_name))
-
logging.info("==== Starting test %s of %s: %s ====",
- current, len(test_topologies), topology_name)
- start_secs = int(time.time())
- try:
- result = run_test(topology_name, classpath, results_checker,
- args, http_server_host_port, update_args, topology_args)
- test_tuple = (topology_name, int(time.time()) - start_secs)
- if isinstance(result, status.TestSuccess):
- successes += [test_tuple]
- elif isinstance(result, status.TestFailure):
- failures += [test_tuple]
- else:
- logging.error("Unrecognized test response returned for test %s: %s",
- topology_name, str(result))
- failures += [test_tuple]
- except status.TestFailure:
- test_tuple = (topology_name, int(time.time()) - start_secs)
- failures += [test_tuple]
+ current, len(test_topologies), topology_name)
+
+ test_threads.append(Thread(target=_run_single_test, args=(topology_name, topology_conf,
+ test_args, http_server_host_port, classpath, update_args, topology_args)))
current += 1
- return successes, failures
+
+ for thread in test_threads:
+ thread.start()
+ for thread in test_threads:
+ thread.join()
+
+ return
def load_result_checker(topology_name, topology_conf,
expected_result_handler, actual_result_handler):
@@ -433,8 +454,10 @@ def main():
logging.error('Unknown argument passed to %s: %s', sys.argv[0], unknown_args[0])
sys.exit(1)
- (successes, failures) = run_tests(conf, args)
+ tests_start_time = int(time.time())
+ run_tests(conf, args)
total = len(failures) + len(successes)
+ logging.info("Total integration test time = %ss" % (int(time.time()) - tests_start_time))
if not failures:
logging.info("SUCCESS: %s (all) tests passed:", len(successes))
diff --git a/integration_test/src/python/topology_test_runner/main.py b/integration_test/src/python/topology_test_runner/main.py
index b12c193e72..f0c2f9da5c 100644
--- a/integration_test/src/python/topology_test_runner/main.py
+++ b/integration_test/src/python/topology_test_runner/main.py
@@ -25,6 +25,7 @@
import time
import uuid
from httplib import HTTPConnection
+from threading import Lock, Thread
from ..common import status
from heron.common.src.python.utils import log
@@ -40,6 +41,8 @@
RETRY_INTERVAL = 10
WAIT_FOR_DEACTIVATION = 5
+successes = []
+failures = []
class TopologyStructureResultChecker(object):
"""
@@ -487,86 +490,62 @@ def cluster_token(cluster, role, env):
# Topology manipulations end
-
-def run_topology_tests(conf, args):
+def run_topology_tests(conf, test_args):
"""
Run the test for each topology specified in the conf file
"""
- successes = []
- failures = []
+ lock = Lock()
timestamp = time.strftime('%Y%m%d%H%M%S')
- http_server_host_port = "%s:%d" % (args.http_hostname, args.http_port)
+ http_server_host_port = "%s:%d" % (test_args.http_hostname, test_args.http_port)
- if args.tests_bin_path.endswith("scala-integration-tests.jar"):
- test_topologies = filter_test_topologies(conf["scalaTopologies"], args.test_topology_pattern)
+ if test_args.tests_bin_path.endswith("scala-integration-tests.jar"):
+ test_topologies = filter_test_topologies(conf["scalaTopologies"], test_args.test_topology_pattern)
topology_classpath_prefix = conf["topologyClasspathPrefix"]
- elif args.tests_bin_path.endswith("integration-topology-tests.jar"):
- test_topologies = filter_test_topologies(conf["javaTopologies"], args.test_topology_pattern)
+ elif test_args.tests_bin_path.endswith("integration-topology-tests.jar"):
+ test_topologies = filter_test_topologies(conf["javaTopologies"], test_args.test_topology_pattern)
topology_classpath_prefix = conf["topologyClasspathPrefix"]
- elif args.tests_bin_path.endswith("heron_integ_topology.pex"):
- test_topologies = filter_test_topologies(conf["pythonTopologies"], args.test_topology_pattern)
+ elif test_args.tests_bin_path.endswith("heron_integ_topology.pex"):
+ test_topologies = filter_test_topologies(conf["pythonTopologies"], test_args.test_topology_pattern)
topology_classpath_prefix = ""
else:
- raise ValueError("Unrecognized binary file type: %s" % args.tests_bin_path)
+ raise ValueError("Unrecognized binary file type: %s" % test_args.tests_bin_path)
processing_type = conf["processingType"]
- current = 1
- for topology_conf in test_topologies:
- topology_name = ("%s_%s_%s") % (timestamp, topology_conf["topologyName"], str(uuid.uuid4()))
- classpath = topology_classpath_prefix + topology_conf["classPath"]
-
- update_args = ""
- deactivate_args = ""
- restart_args = ""
- topology_args = ""
- if "updateArgs" in topology_conf:
- update_args = topology_conf["updateArgs"]
- if "deactivateArgs" in topology_conf:
- deactivate_args = True
- if "restartArgs" in topology_conf:
- restart_args = True
-
- if "topologyArgs" in topology_conf:
- topology_args = "%s %s" % (topology_args, topology_conf["topologyArgs"])
-
- expected_topo_result_file_path = \
- args.topologies_path + "/" + topology_conf["expectedTopoResultRelativePath"]
- if "expectedStateResultRelativePath" in topology_conf:
- expected_state_result_file_path = \
- args.topologies_path + "/" + topology_conf["expectedStateResultRelativePath"]
-
+ def _run_single_test(topology_name, topology_conf, test_args, http_server_host_port, classpath,
+ update_args, deactivate_args, restart_args, topology_args, expected_topo_result_file_path,
+ expected_state_result_file_path):
+ global successes, failures
check_type = topology_conf["checkType"]
if check_type == 'topology_structure':
results_checker = load_result_checker(
check_type, topology_name,
FileBasedExpectedResultsHandler(expected_topo_result_file_path),
- ZkFileBasedActualResultsHandler(topology_name, args.cluster))
+ ZkFileBasedActualResultsHandler(topology_name, test_args.cluster))
elif check_type == 'checkpoint_state':
if processing_type == 'stateful':
results_checker = load_result_checker(
check_type, topology_name,
FileBasedExpectedResultsHandler(expected_topo_result_file_path),
- ZkFileBasedActualResultsHandler(topology_name, args.cluster),
+ ZkFileBasedActualResultsHandler(topology_name, test_args.cluster),
FileBasedExpectedResultsHandler(expected_state_result_file_path),
HttpBasedActualResultsHandler(http_server_host_port, topology_name))
elif processing_type == 'non_stateful':
raise ValueError("Cannot check instance checkpoint state in non_stateful processing. "
- + "Not running topology: " + topology_name)
+ + "Not running topology: " + topology_name)
else:
raise ValueError("Unrecognized processing type for topology: " + topology_name)
else:
raise ValueError("Unrecognized check type for topology: " + topology_name)
- logging.info("==== Starting test %s of %s: %s ====",
- current, len(test_topologies), topology_name)
start_secs = int(time.time())
try:
result = run_topology_test(topology_name, classpath, results_checker,
- args, update_args, deactivate_args, restart_args, http_server_host_port, topology_args,
+ test_args, update_args, deactivate_args, restart_args, http_server_host_port, topology_args,
check_type)
test_tuple = (topology_name, int(time.time()) - start_secs)
+ lock.acquire()
if isinstance(result, status.TestSuccess):
successes += [test_tuple]
elif isinstance(result, status.TestFailure):
@@ -575,12 +554,55 @@ def run_topology_tests(conf, args):
logging.error("Unrecognized test response returned for test %s: %s",
topology_name, str(result))
failures += [test_tuple]
+ lock.release()
except status.TestFailure:
test_tuple = (topology_name, int(time.time()) - start_secs)
+ lock.acquire()
failures += [test_tuple]
+ lock.release()
+
+ current = 1
+ test_threads = []
+ for topology_conf in test_topologies:
+ topology_name = ("%s_%s_%s") % (timestamp, topology_conf["topologyName"], str(uuid.uuid4()))
+ classpath = topology_classpath_prefix + topology_conf["classPath"]
+
+ update_args = ""
+ deactivate_args = ""
+ restart_args = ""
+ topology_args = ""
+ if "updateArgs" in topology_conf:
+ update_args = topology_conf["updateArgs"]
+ if "deactivateArgs" in topology_conf:
+ deactivate_args = True
+ if "restartArgs" in topology_conf:
+ restart_args = True
+
+ if "topologyArgs" in topology_conf:
+ topology_args = "%s %s" % (topology_args, topology_conf["topologyArgs"])
+
+ expected_topo_result_file_path = \
+ test_args.topologies_path + "/" + topology_conf["expectedTopoResultRelativePath"]
+ expected_state_result_file_path = ''
+ if "expectedStateResultRelativePath" in topology_conf:
+ expected_state_result_file_path = \
+ test_args.topologies_path + "/" + topology_conf["expectedStateResultRelativePath"]
+
+ logging.info("==== Starting test %s of %s: %s ====",
+ current, len(test_topologies), topology_name)
+
+ test_threads.append(Thread(target=_run_single_test, args=(topology_name, topology_conf, test_args,
+ http_server_host_port, classpath, update_args, deactivate_args, restart_args,
+ topology_args, expected_topo_result_file_path, expected_state_result_file_path)))
current += 1
- return successes, failures
+
+ for thread in test_threads:
+ thread.start()
+ for thread in test_threads:
+ thread.join()
+
+ return
def load_result_checker(check_type, topology_name,
@@ -638,8 +660,10 @@ def main():
logging.error('Unknown argument passed to %s: %s', sys.argv[0], unknown_args[0])
sys.exit(1)
- (successes, failures) = run_topology_tests(conf, args)
+ tests_start_time = int(time.time())
+ run_topology_tests(conf, args)
total = len(failures) + len(successes)
+ logging.info("Total integration topology test time = %ss" % (int(time.time()) - tests_start_time))
if not failures:
logging.info("SUCCESS: %s (all) tests passed:", len(successes))
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services