[impala] branch master updated: IMPALA-8820: fix start-impala-cluster catalogd startup

[ta...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git


The following commit(s) were added to refs/heads/master by this push:
     new 615a821  IMPALA-8820: fix start-impala-cluster catalogd startup
615a821 is described below

commit 615a8213159268aa893da51921d3c7728d82c7f7
Author: Tim Armstrong <ta...@cloudera.com>
AuthorDate: Wed Jul 31 13:22:58 2019 -0700

    IMPALA-8820: fix start-impala-cluster catalogd startup
    
    The catalogd process sometimes changes its name to "main"
    after an ubuntu 16.04 update.
    
    This avoids the issue by checking the first element of the
    command line instead, which should reflect the binary
    that was executed more reliably.
    
    Testing:
    This failed consistently before the change and now passes consistently
    on my development machine.
    
    Change-Id: Ib9396669481e4194beb6247c8d8b6064cb5119bb
    Reviewed-on: http://gerrit.cloudera.org:8080/13971
    Reviewed-by: Joe McDonnell <jo...@cloudera.com>
    Tested-by: Tim Armstrong <ta...@cloudera.com>
---
 bin/start-impala-cluster.py    |  4 ++--
 tests/common/impala_cluster.py | 21 +++++++++++++++------
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/bin/start-impala-cluster.py b/bin/start-impala-cluster.py
index 43294d6..75e8b93 100755
--- a/bin/start-impala-cluster.py
+++ b/bin/start-impala-cluster.py
@@ -154,7 +154,7 @@ def check_process_exists(binary, attempts=1):
   otherwise.
   """
   for _ in range(attempts):
-    for proc in find_user_processes([binary]):
+    for _ in find_user_processes([binary]):
       return True
     sleep(1)
   return False
@@ -181,7 +181,7 @@ def build_java_tool_options(jvm_debug_port=None):
 def kill_matching_processes(binary_names, force=False):
   """Kills all processes with the given binary name, waiting for them to exit"""
   # Send all the signals before waiting so that processes can clean up in parallel.
-  processes = list(find_user_processes(binary_names))
+  processes = [proc for _, proc in find_user_processes(binary_names)]
   for process in processes:
     try:
       if force:
diff --git a/tests/common/impala_cluster.py b/tests/common/impala_cluster.py
index 6136091..b70d9dd 100644
--- a/tests/common/impala_cluster.py
+++ b/tests/common/impala_cluster.py
@@ -209,7 +209,7 @@ class ImpalaCluster(object):
     impalads = list()
     statestored = list()
     catalogd = None
-    for process in find_user_processes(['impalad', 'catalogd', 'statestored']):
+    for binary, process in find_user_processes(['impalad', 'catalogd', 'statestored']):
       # IMPALA-6889: When a process shuts down and becomes a zombie its cmdline becomes
       # empty for a brief moment, before it gets reaped by its parent (see man proc). We
       # copy the cmdline to prevent it from changing between the following checks and
@@ -223,11 +223,11 @@ class ImpalaCluster(object):
         continue
       if len(cmdline) == 0:
         continue
-      if process.name == 'impalad':
+      if binary == 'impalad':
         impalads.append(ImpaladProcess(cmdline))
-      elif process.name == 'statestored':
+      elif binary == 'statestored':
         statestored.append(StateStoreProcess(cmdline))
-      elif process.name == 'catalogd':
+      elif binary == 'catalogd':
         catalogd = CatalogdProcess(cmdline)
 
     self.__sort_impalads(impalads)
@@ -532,11 +532,20 @@ class CatalogdProcess(BaseImpalaProcess):
 
 def find_user_processes(binaries):
   """Returns an iterator over all processes owned by the current user with a matching
-  binary name from the provided list."""
+  binary name from the provided list. Return a iterable of tuples, with each tuple
+  containing the binary name and the psutil.Process object."""
   for pid in psutil.get_pid_list():
     try:
       process = psutil.Process(pid)
-      if process.username == getuser() and process.name in binaries: yield process
+      cmdline = process.cmdline
+      if process.username != getuser() or len(cmdline) == 0:
+        continue
+      # IMPALA-8820 - sometimes the process name does not reflect the executed binary
+      # because the process can change its own name at runtime. Checking the command
+      # line is more robust.
+      binary_name = os.path.basename(cmdline[0])
+      if binary_name in binaries:
+        yield binary_name, process
     except KeyError, e:
       if "uid not found" not in str(e):
         raise