You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by as...@apache.org on 2024/04/16 16:08:11 UTC
(impala) branch master updated (61ceb16d8 -> 0606fc760)
This is an automated email from the ASF dual-hosted git repository.
asherman pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from 61ceb16d8 IMPALA-12999: Add log4j.properties to the DEB/RPM packages
new 74ff59b91 IMPALA-12963: Return parent PID when children spawned
new 0606fc760 IMPALA-11495: Add glibc version and effective locale to the Web UI
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/util/default-path-handlers.cc | 6 +++
tests/common/impala_cluster.py | 54 ++++++++++++++++----------
tests/custom_cluster/test_query_log.py | 69 ++++++++++++++++------------------
tests/webserver/test_web_pages.py | 8 ++++
www/root.tmpl | 3 ++
5 files changed, 84 insertions(+), 56 deletions(-)
(impala) 01/02: IMPALA-12963: Return parent PID when children spawned
Posted by as...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asherman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 74ff59b9138f325fd22ce198bd01423abafd3688
Author: Michael Smith <mi...@cloudera.com>
AuthorDate: Tue Apr 9 13:19:19 2024 -0700
IMPALA-12963: Return parent PID when children spawned
Returns the original PID for a command rather than any children that may
be active. This happens during graceful shutdown in UBSAN tests. Also
updates 'kill' to use the version of 'get_pid' that logs details to help
with debugging.
Moves try block in test_query_log.py to after client2 has been
initialized. Removes 'drop table' on unique_database, since test suite
already handles cleanup.
Change-Id: I214e79507c717340863d27f68f6ea54c169e4090
Reviewed-on: http://gerrit.cloudera.org:8080/21278
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
tests/common/impala_cluster.py | 54 ++++++++++++++++----------
tests/custom_cluster/test_query_log.py | 69 ++++++++++++++++------------------
2 files changed, 67 insertions(+), 56 deletions(-)
diff --git a/tests/common/impala_cluster.py b/tests/common/impala_cluster.py
index 51fefa715..1465ab31b 100644
--- a/tests/common/impala_cluster.py
+++ b/tests/common/impala_cluster.py
@@ -401,7 +401,7 @@ class Process(object):
"""Gets the PIDs of the process. In some circumstances, a process can run multiple
times, e.g. when it forks in the Breakpad crash handler. Returns an empty list if no
PIDs can be determined."""
- pids = self.__get_pids()
+ pids = [proc['pid'] for proc in self.__get_procs()]
if pids:
LOG.info("Found PIDs %s for %s" % (", ".join(map(str, pids)), " ".join(self.cmd)))
else:
@@ -409,37 +409,53 @@ class Process(object):
" ".join(self.cmd))
return pids
- def __get_pid(self):
- pids = self.__get_pids()
- assert len(pids) < 2, "Expected single pid but found %s" % ", ".join(map(str, pids))
- return len(pids) == 1 and pids[0] or None
+ def __procs_str(self, procs):
+ return "\n".join([str(proc) for proc in procs])
- def __get_pids(self):
+ def __get_pid(self):
+ procs = self.__get_procs()
+ # Return early for containerized environments
+ if len(procs) == 1:
+ return procs[0]['pid']
+
+ result = None
+ # In some circumstances - notably ubsan tests - child processes can be slow to exit.
+ # Only return the original process, i.e. one who's parent has a different cmd.
+ pids = [proc['pid'] for proc in procs]
+ for process in procs:
+ if process['ppid'] not in pids:
+ assert result is None,\
+ "Multiple non-child processes:\n%s" % self.__procs_str(procs)
+ result = process['pid']
+ else:
+ LOG.info("Child process active:\n%s" % self.__procs_str(procs))
+
+ return result
+
+ def __get_procs(self):
+ """
+ Returns a list of dicts containing {pid, ppid, cmdline} for related processes.
+ """
if self.container_id is not None:
container_info = get_container_info(self.container_id)
if container_info["State"]["Status"] != "running":
return []
- return [container_info["State"]["Pid"]]
+ return [{'pid': container_info["State"]["Pid"], 'ppid': 0, 'cmdline': self.cmd}]
# In non-containerised case, search for process based on matching command lines.
- pids = []
- for pid in psutil.pids():
- try:
- process = psutil.Process(pid)
- if set(self.cmd) == set(process.cmdline()):
- pids.append(pid)
- except psutil.NoSuchProcess:
- # A process from psutil.pids() no longer exists, continue. We don't log this
- # error since it can refer to arbitrary processes outside of our testing code.
- pass
- return pids
+ procs = []
+ for process in psutil.process_iter(['pid', 'ppid', 'cmdline']):
+ # Use info because it won't throw NoSuchProcess exceptions.
+ if set(self.cmd) == set(process.info['cmdline']):
+ procs.append(process.info)
+ return procs
def kill(self, signal=SIGKILL):
"""
Kills the given processes.
"""
if self.container_id is None:
- pid = self.__get_pid()
+ pid = self.get_pid()
assert pid is not None, "No processes for %s" % self
LOG.info('Killing %s with signal %s' % (self, signal))
exec_process("kill -%d %d" % (signal, pid))
diff --git a/tests/custom_cluster/test_query_log.py b/tests/custom_cluster/test_query_log.py
index e02993217..dd61e7291 100644
--- a/tests/custom_cluster/test_query_log.py
+++ b/tests/custom_cluster/test_query_log.py
@@ -56,13 +56,11 @@ class TestQueryLogTableBase(CustomClusterTestSuite):
cls.ImpalaTestMatrix.add_dimension(ImpalaTestDimension('protocol',
cls.PROTOCOL_BEESWAX, cls.PROTOCOL_HS2))
- def get_client(self, protocol, query_table_name=""):
+ def get_client(self, protocol, query_table_name=QUERY_TBL):
"""Retrieves the default Impala client for the specified protocol. This client is
automatically closed after the test completes. Also ensures the completed queries
table has been successfully created by checking the logs to verify the create
table sql has finished."""
- if query_table_name == "":
- query_table_name = self.QUERY_TBL
# These tests run very quickly and can actually complete before Impala has finished
# creating the completed queries table. Thus, to make these tests more robust, this
@@ -704,24 +702,24 @@ class TestQueryLogTableHS2(TestQueryLogTableBase):
impalad = self.cluster.get_first_impalad()
client = self.get_client(vector.get_value('protocol'))
- try:
- # Execute sql statements to ensure all get written to the query log table.
- sql1 = client.execute("select 1")
- assert sql1.success
+ # Execute sql statements to ensure all get written to the query log table.
+ sql1 = client.execute("select 1")
+ assert sql1.success
- sql2 = client.execute("select 2")
- assert sql2.success
+ sql2 = client.execute("select 2")
+ assert sql2.success
- sql3 = client.execute("select 3")
- assert sql3.success
+ sql3 = client.execute("select 3")
+ assert sql3.success
- impalad.service.wait_for_metric_value("impala-server.completed-queries.queued", 3,
- 60)
+ impalad.service.wait_for_metric_value("impala-server.completed-queries.queued", 3,
+ 60)
- impalad.kill_and_wait_for_exit(SIGRTMIN)
+ impalad.kill_and_wait_for_exit(SIGRTMIN)
- client2 = self.create_client_for_nth_impalad(1, vector.get_value('protocol'))
+ client2 = self.create_client_for_nth_impalad(1, vector.get_value('protocol'))
+ try:
def assert_func(last_iteration):
results = client2.execute("select query_id,sql from {0} where query_id in "
"('{1}','{2}','{3}')".format(self.QUERY_TBL,
@@ -783,30 +781,27 @@ class TestQueryLogTableAll(TestQueryLogTableBase):
tbl_name = "{0}.{1}".format(unique_database, unique_name)
client = self.get_client(vector.get_value('protocol'))
- try:
- # Create the test table.
- create_tbl_sql = "create table {0} (id INT, product_name STRING) " \
- "partitioned by (category INT)".format(tbl_name)
- create_tbl_results = client.execute(create_tbl_sql)
- assert create_tbl_results.success
-
- insert_sql = "insert into {0} (id,category,product_name) values " \
- "(0,1,'the product')".format(tbl_name)
- res = client.execute(insert_sql, fetch_profile_after_close=True)
- assert res.success
+ # Create the test table.
+ create_tbl_sql = "create table {0} (id INT, product_name STRING) " \
+ "partitioned by (category INT)".format(tbl_name)
+ create_tbl_results = client.execute(create_tbl_sql)
+ assert create_tbl_results.success
- # Include the two queries run by the unique_database fixture setup.
- self.cluster.get_first_impalad().service.wait_for_metric_value(
- "impala-server.completed-queries.written", 4, 60)
-
- client2 = self.create_client_for_nth_impalad(2, vector.get_value('protocol'))
- try:
- assert client2 is not None
- assert_query(self.QUERY_TBL, client2, "test_query_hist_3", res.runtime_profile)
- finally:
- client2.close()
+ insert_sql = "insert into {0} (id,category,product_name) values " \
+ "(0,1,'the product')".format(tbl_name)
+ res = client.execute(insert_sql, fetch_profile_after_close=True)
+ assert res.success
+
+ # Include the two queries run by the unique_database fixture setup.
+ self.cluster.get_first_impalad().service.wait_for_metric_value(
+ "impala-server.completed-queries.written", 4, 60)
+
+ client2 = self.create_client_for_nth_impalad(2, vector.get_value('protocol'))
+ try:
+ assert client2 is not None
+ assert_query(self.QUERY_TBL, client2, "test_query_hist_3", res.runtime_profile)
finally:
- client.execute("drop table if exists {0}".format(tbl_name))
+ client2.close()
@CustomClusterTestSuite.with_args(impalad_args="--enable_workload_mgmt "
"--query_log_write_interval_s=1 "
(impala) 02/02: IMPALA-11495: Add glibc version and effective locale to the Web UI
Posted by as...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asherman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 0606fc760f21587206cfb4f8256c7cd575050cf2
Author: Saurabh Katiyal <sa...@gmail.com>
AuthorDate: Sun Apr 7 17:16:31 2024 +0530
IMPALA-11495: Add glibc version and effective locale to the Web UI
Added a new section "Other Info" in root page for WebUI,
displaying effective locale and glibc version.
Change-Id: Ia69c4d63df4beae29f5261691a8dcdd04b931de7
Reviewed-on: http://gerrit.cloudera.org:8080/21252
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/util/default-path-handlers.cc | 6 ++++++
tests/webserver/test_web_pages.py | 8 ++++++++
www/root.tmpl | 3 +++
3 files changed, 17 insertions(+)
diff --git a/be/src/util/default-path-handlers.cc b/be/src/util/default-path-handlers.cc
index 8bd4edaca..6ee358cc3 100644
--- a/be/src/util/default-path-handlers.cc
+++ b/be/src/util/default-path-handlers.cc
@@ -23,6 +23,7 @@
#include <boost/algorithm/string.hpp>
#include <boost/bind.hpp>
#include <gutil/strings/substitute.h>
+#include <gnu/libc-version.h>
#include "common/logging.h"
#include "kudu/util/flags.h"
@@ -251,6 +252,11 @@ void RootHandler(const Webserver::WebRequest& req, Document* document) {
Value cgroup_info(CGroupUtil::DebugString().c_str(), document->GetAllocator());
document->AddMember("cgroup_info", cgroup_info, document->GetAllocator());
+ Value effective_locale(std::locale("").name().c_str(), document->GetAllocator());
+ document->AddMember("effective_locale", effective_locale, document->GetAllocator());
+ Value glibc_version(gnu_get_libc_version(), document->GetAllocator());
+ document->AddMember("glibc_version", glibc_version, document->GetAllocator());
+
if (CommonMetrics::PROCESS_START_TIME != nullptr) {
Value process_start_time(
CommonMetrics::PROCESS_START_TIME->GetValue().c_str(), document->GetAllocator());
diff --git a/tests/webserver/test_web_pages.py b/tests/webserver/test_web_pages.py
index 61e8990c9..5e4a73118 100644
--- a/tests/webserver/test_web_pages.py
+++ b/tests/webserver/test_web_pages.py
@@ -115,6 +115,14 @@ class TestWebPage(ImpalaTestSuite):
if build_flags["cmake_build_type"] in ["debug"]:
assert not is_ndebug
+ def test_root_other_info(self):
+ """Tests to check glibc version and locale is available"""
+ for port in self.TEST_PORTS_WITH_SS:
+ other_info_page = requests.get(self.ROOT_URL.format(port) + "/?json").text
+ other_info = json.loads(other_info_page)
+ assert "effective_locale" in other_info
+ assert "glibc_version" in other_info
+
def test_memz(self):
"""Tests /memz at impalad / statestored / catalogd"""
diff --git a/www/root.tmpl b/www/root.tmpl
index ef42be6ab..da779917d 100644
--- a/www/root.tmpl
+++ b/www/root.tmpl
@@ -53,4 +53,7 @@ this page is rendered -->
<h2>CGroup Info</h2>
<pre>{{cgroup_info}}</pre>
+ <h2>Other Info</h2>
+ <pre>Locale: {{effective_locale}}<br>GLIBC Version: {{glibc_version}}</pre>
+
{{>www/common-footer.tmpl}}