incubator-impala git commit: IMPALA-2605: Omit the sort and mini stress tests

[jb...@apache.org]

Repository: incubator-impala
Updated Branches:
  refs/heads/master c4ee03a7e -> 9fa2ff713


IMPALA-2605: Omit the sort and mini stress tests

These stress tests were sometimes causing the end-to-end tests to hang
indefinitey, including in the pre-merge testing (sometimes called
"GVO" or "GVM").

This patch also prints to stdout some connections metrics that may
prove useful for debugging stress test hangs in the future. The
metrics are printed before and after stress tests are run when
run-tests.py is used.

Change-Id: Ibd30abf8215415e0f2830b725e43b005daa2bb2d
Reviewed-on: http://gerrit.cloudera.org:8080/5401
Reviewed-by: Jim Apple <jb...@apache.org>
Tested-by: Impala Public Jenkins


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/9fa2ff71
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/9fa2ff71
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/9fa2ff71

Branch: refs/heads/master
Commit: 9fa2ff7138c5385277aec128b74d079aff902617
Parents: c4ee03a
Author: Jim Apple <jb...@apache.org>
Authored: Wed Dec 7 11:37:46 2016 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Fri Jan 6 21:32:15 2017 +0000

----------------------------------------------------------------------
 tests/run-tests.py               | 20 ++++++++++++++++++++
 tests/stress/test_mini_stress.py |  4 ++++
 2 files changed, 24 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9fa2ff71/tests/run-tests.py
----------------------------------------------------------------------
diff --git a/tests/run-tests.py b/tests/run-tests.py
index 7acc7a9..fc23338 100755
--- a/tests/run-tests.py
+++ b/tests/run-tests.py
@@ -20,7 +20,10 @@
 # Runs the Impala query tests, first executing the tests that cannot be run in parallel
 # and then executing the remaining tests in parallel. All additional command line options
 # are passed to py.test.
+from tests.common.impala_cluster import ImpalaCluster
+from tests.common.impala_service import ImpaladService
 import itertools
+import json
 import multiprocessing
 import os
 import pytest
@@ -149,6 +152,21 @@ def build_ignore_dir_arg_list(valid_dirs):
                    for d in set(subdirs) - set(valid_dirs)])
 
 
+def print_metrics(substring):
+  """Prints metrics with the given substring in the name"""
+  for impalad in ImpalaCluster().impalads:
+    print ">" * 80
+    port = impalad._get_webserver_port()
+    print "connections metrics for impalad at port {}:".format(port)
+    debug_info = json.loads(ImpaladService(
+            impalad.hostname,
+            webserver_port=port)
+            .open_debug_webpage('metrics?json').read())
+    for metric in debug_info['metric_group']['metrics']:
+      if substring in metric['name']:
+        print json.dumps(metric, indent=1)
+    print "<" * 80
+
 if __name__ == "__main__":
   exit_on_error = '-x' in sys.argv or '--exitfirst' in sys.argv
   test_executor = TestExecutor(exit_on_error=exit_on_error)
@@ -169,9 +187,11 @@ if __name__ == "__main__":
   test_executor.run_tests(args)
 
   # Run the stress tests tests
+  print_metrics('connections')
   args = '-m "stress" -n %d %s' %\
       (NUM_STRESS_CLIENTS, build_test_args('stress', VALID_TEST_DIRS))
   test_executor.run_tests(args)
+  print_metrics('connections')
 
   # Run the remaining query tests in parallel
   args = '-m "not execute_serially and not stress"  -n %d %s' %\

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9fa2ff71/tests/stress/test_mini_stress.py
----------------------------------------------------------------------
diff --git a/tests/stress/test_mini_stress.py b/tests/stress/test_mini_stress.py
index fd4a3a7..87a8742 100644
--- a/tests/stress/test_mini_stress.py
+++ b/tests/stress/test_mini_stress.py
@@ -46,11 +46,15 @@ class TestMiniStress(ImpalaTestSuite):
       cls.TestMatrix.add_constraint(lambda v:\
           v.get_value('exec_option')['batch_size'] != 1)
 
+  @pytest.mark.xfail(run=False, reason="IMPALA-2605: the stress tests have a history of "
+                     "causing the end-to-end tests to hang")
   @pytest.mark.stress
   def test_mini_stress(self, vector):
     for i in xrange(NUM_ITERATIONS):
       self.run_test_case('stress', vector)
 
+  @pytest.mark.xfail(run=False, reason="IMPALA-2605: the stress tests have a history of "
+                     "causing the end-to-end tests to hang")
   @pytest.mark.stress
   def test_sort_stress(self, vector):
     if self.exploration_strategy() == 'core':