You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2019/02/10 20:24:54 UTC
[impala] 08/09: IMPALA-8163: Add banner showing local catalog mode
on web UI when enabled.
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 81305f205849f20fe3d31a142b10aee16d5c46de
Author: Anurag Mantripragada <an...@gmail.com>
AuthorDate: Thu Feb 7 14:18:26 2019 -0800
IMPALA-8163: Add banner showing local catalog mode on web UI
when enabled.
Added a banner on coordinator web UI for visibility of local
catalog mode.
Testing:
1. Added a test in test_local_catalog to scrape webpages and
search for strings. Tested banner appears when local catalog
mode is enabled.
2. Refactored parts of test_local_catalog into
TestObservability class.
Change-Id: Iaeb2c7b20742a2630d9509da9b629d5f45c55207
Reviewed-on: http://gerrit.cloudera.org:8080/12413
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/util/default-path-handlers.cc | 3 +
tests/custom_cluster/test_local_catalog.py | 128 +++++++++++++++--------------
www/root.tmpl | 3 +-
3 files changed, 72 insertions(+), 62 deletions(-)
diff --git a/be/src/util/default-path-handlers.cc b/be/src/util/default-path-handlers.cc
index 4458b83..6beb148 100644
--- a/be/src/util/default-path-handlers.cc
+++ b/be/src/util/default-path-handlers.cc
@@ -48,6 +48,7 @@ using namespace rapidjson;
using namespace strings;
DECLARE_bool(enable_process_lifetime_heap_profiling);
+DECLARE_bool(use_local_catalog);
DEFINE_int64(web_log_bytes, 1024 * 1024,
"The maximum number of bytes to display on the debug webserver's log page");
@@ -285,6 +286,8 @@ void RootHandler(const Webserver::ArgumentMap& args, Document* document) {
document->AddMember("impala_server_mode", true, document->GetAllocator());
document->AddMember("is_coordinator", impala_server->IsCoordinator(),
document->GetAllocator());
+ document->AddMember("use_local_catalog", FLAGS_use_local_catalog,
+ document->GetAllocator());
document->AddMember("is_executor", impala_server->IsExecutor(),
document->GetAllocator());
bool is_quiescing = impala_server->IsShuttingDown();
diff --git a/tests/custom_cluster/test_local_catalog.py b/tests/custom_cluster/test_local_catalog.py
index f9399d7..6173bde 100644
--- a/tests/custom_cluster/test_local_catalog.py
+++ b/tests/custom_cluster/test_local_catalog.py
@@ -29,21 +29,6 @@ RETRY_PROFILE_MSG = 'Retried query planning due to inconsistent metadata'
class TestCompactCatalogUpdates(CustomClusterTestSuite):
- def get_catalog_cache_metrics(self, impalad):
- """ Returns catalog cache metrics as a dict by scraping the json metrics page on the
- given impalad"""
- child_groups =\
- impalad.service.get_debug_webpage_json('metrics')['metric_group']['child_groups']
- for group in child_groups:
- if group['name'] != 'impala-server': continue
- # Filter catalog cache metrics.
- for child_group in group['child_groups']:
- if child_group['name'] != 'catalog': continue
- metrics_data = [(metric['name'], metric['value'])
- for metric in child_group['metrics'] if 'catalog.cache' in metric['name']]
- return dict(metrics_data)
- assert False, "Catalog cache metrics not found in %s" % child_groups
-
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
impalad_args="--use_local_catalog=true",
@@ -167,52 +152,6 @@ class TestCompactCatalogUpdates(CustomClusterTestSuite):
finally:
client.close()
- @pytest.mark.execute_serially
- @CustomClusterTestSuite.with_args(
- impalad_args="--use_local_catalog=true",
- catalogd_args="--catalog_topic_mode=minimal")
- def test_cache_metrics(self, unique_database):
- """
- Test that profile output includes impalad local cache metrics. Also verifies that
- the daemon level metrics are updated between query runs.
- """
- try:
- impalad = self.cluster.impalads[0]
- client = impalad.service.create_beeswax_client()
- cache_hit_rate_metric_key = "catalog.cache.hit-rate"
- cache_miss_rate_metric_key = "catalog.cache.miss-rate"
- cache_hit_count_metric_key = "catalog.cache.hit-count"
- cache_request_count_metric_key = "catalog.cache.request-count"
- cache_request_count_prev_run = 0
- cache_hit_count_prev_run = 0
- test_table_name = "%s.test_cache_metrics_test_tbl" % unique_database
- # A mix of queries of various types.
- queries_to_test = ["select count(*) from functional.alltypes",
- "explain select count(*) from functional.alltypes",
- "create table %s (a int)" % test_table_name,
- "drop table %s" % test_table_name]
- for _ in xrange(0, 10):
- for query in queries_to_test:
- ret = self.execute_query_expect_success(client, query)
- assert ret.runtime_profile.count("Frontend:") == 1
- assert ret.runtime_profile.count("CatalogFetch") > 1
- cache_metrics = self.get_catalog_cache_metrics(impalad)
- cache_hit_rate = cache_metrics[cache_hit_rate_metric_key]
- cache_miss_rate = cache_metrics[cache_miss_rate_metric_key]
- cache_hit_count = cache_metrics[cache_hit_count_metric_key]
- cache_request_count = cache_metrics[cache_request_count_metric_key]
- assert cache_hit_rate > 0.0 and cache_hit_rate < 1.0
- assert cache_miss_rate > 0.0 and cache_miss_rate < 1.0
- assert cache_hit_count > cache_hit_count_prev_run,\
- "%s not updated between two query runs, query - %s"\
- % (cache_hit_count_metric_key, query)
- assert cache_request_count > cache_request_count_prev_run,\
- "%s not updated betweeen two query runs, query - %s"\
- % (cache_request_count_metric_key, query)
- cache_hit_count_prev_run = cache_hit_count
- cache_request_count_prev_run = cache_request_count
- finally:
- client.close()
class TestLocalCatalogRetries(CustomClusterTestSuite):
@@ -373,3 +312,70 @@ class TestLocalCatalogRetries(CustomClusterTestSuite):
finally:
client1.close()
client2.close()
+
+
+class TestObservability(CustomClusterTestSuite):
+
+ def get_catalog_cache_metrics(self, impalad):
+ """ Returns catalog cache metrics as a dict by scraping the json metrics page on the
+ given impalad"""
+ child_groups =\
+ impalad.service.get_debug_webpage_json('metrics')['metric_group']['child_groups']
+ for group in child_groups:
+ if group['name'] != 'impala-server': continue
+ # Filter catalog cache metrics.
+ for child_group in group['child_groups']:
+ if child_group['name'] != 'catalog': continue
+ metrics_data = [(metric['name'], metric['value'])
+ for metric in child_group['metrics'] if 'catalog.cache' in metric['name']]
+ return dict(metrics_data)
+ assert False, "Catalog cache metrics not found in %s" % child_groups
+
+ @pytest.mark.execute_serially
+ @CustomClusterTestSuite.with_args(
+ impalad_args="--use_local_catalog=true",
+ catalogd_args="--catalog_topic_mode=minimal")
+ def test_cache_metrics(self, unique_database):
+ """
+ Test that profile output includes impalad local cache metrics. Also verifies that
+ the daemon level metrics are updated between query runs.
+ """
+ try:
+ impalad = self.cluster.impalads[0]
+ # Make sure local catalog mode is enabled and visible on web UI.
+ assert '(Local Catalog Mode)' in impalad.service.read_debug_webpage('/')
+ client = impalad.service.create_beeswax_client()
+ cache_hit_rate_metric_key = "catalog.cache.hit-rate"
+ cache_miss_rate_metric_key = "catalog.cache.miss-rate"
+ cache_hit_count_metric_key = "catalog.cache.hit-count"
+ cache_request_count_metric_key = "catalog.cache.request-count"
+ cache_request_count_prev_run = 0
+ cache_hit_count_prev_run = 0
+ test_table_name = "%s.test_cache_metrics_test_tbl" % unique_database
+ # A mix of queries of various types.
+ queries_to_test = ["select count(*) from functional.alltypes",
+ "explain select count(*) from functional.alltypes",
+ "create table %s (a int)" % test_table_name,
+ "drop table %s" % test_table_name]
+ for _ in xrange(0, 10):
+ for query in queries_to_test:
+ ret = self.execute_query_expect_success(client, query)
+ assert ret.runtime_profile.count("Frontend:") == 1
+ assert ret.runtime_profile.count("CatalogFetch") > 1
+ cache_metrics = self.get_catalog_cache_metrics(impalad)
+ cache_hit_rate = cache_metrics[cache_hit_rate_metric_key]
+ cache_miss_rate = cache_metrics[cache_miss_rate_metric_key]
+ cache_hit_count = cache_metrics[cache_hit_count_metric_key]
+ cache_request_count = cache_metrics[cache_request_count_metric_key]
+ assert cache_hit_rate > 0.0 and cache_hit_rate < 1.0
+ assert cache_miss_rate > 0.0 and cache_miss_rate < 1.0
+ assert cache_hit_count > cache_hit_count_prev_run,\
+ "%s not updated between two query runs, query - %s"\
+ % (cache_hit_count_metric_key, query)
+ assert cache_request_count > cache_request_count_prev_run,\
+ "%s not updated betweeen two query runs, query - %s"\
+ % (cache_request_count_metric_key, query)
+ cache_hit_count_prev_run = cache_hit_count
+ cache_request_count_prev_run = cache_request_count
+ finally:
+ client.close()
diff --git a/www/root.tmpl b/www/root.tmpl
index a33df9b..ef42be6 100644
--- a/www/root.tmpl
+++ b/www/root.tmpl
@@ -27,7 +27,8 @@ under the License.
{{/is_quiescing}}
{{?impala_server_mode}}
- <h2>Impala Server Mode: {{?is_coordinator}}Coordinator{{/is_coordinator}}
+ <h2>Impala Server Mode: {{?is_coordinator}}Coordinator{{?use_local_catalog}}
+ (Local Catalog Mode){{/use_local_catalog}}{{/is_coordinator}}
{{?is_executor}}Executor{{/is_executor}}</h2>
{{/impala_server_mode}}