You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2019/08/20 17:49:38 UTC
[impala] branch master updated: IMPALA-7935: Disable
/catalog_object in local catalog mode.
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new bc47810 IMPALA-7935: Disable /catalog_object in local catalog mode.
bc47810 is described below
commit bc4781078717a191dcac8e2c284d472ba524abc9
Author: Anurag Mantripragada <an...@gmail.com>
AuthorDate: Thu Aug 8 14:40:04 2019 -0700
IMPALA-7935: Disable /catalog_object in local catalog mode.
getTCatalogObject() is not supported in local catalog mode
since metadata is partially fetched on demand. Removed hyperlinks
to the /catalog_object endpoints when local_catalog_mode is enabled.
Testing:
Added a test to test_local_catalog::TestObservability to verify
/catalog_mode endpoint is disabled when in local catalog mode.
Change-Id: Ia04797b32964c2edaa2e860dcf510d6f9cccd81c
Reviewed-on: http://gerrit.cloudera.org:8080/12443
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/service/impala-http-handler.cc | 22 ++++++++++----
tests/custom_cluster/test_local_catalog.py | 3 ++
tests/webserver/test_web_pages.py | 46 ++++++++++++++++++++++--------
www/catalog.tmpl | 29 ++++++++++++++++---
4 files changed, 79 insertions(+), 21 deletions(-)
diff --git a/be/src/service/impala-http-handler.cc b/be/src/service/impala-http-handler.cc
index 20e0716..22bd189 100644
--- a/be/src/service/impala-http-handler.cc
+++ b/be/src/service/impala-http-handler.cc
@@ -59,6 +59,7 @@ using namespace rapidjson;
using namespace strings;
DECLARE_int32(query_log_size);
+DECLARE_bool(use_local_catalog);
namespace {
@@ -104,8 +105,12 @@ void ImpalaHttpHandler::RegisterHandlers(Webserver* webserver) {
webserver->RegisterUrlCallback("/catalog", "catalog.tmpl",
MakeCallback(this, &ImpalaHttpHandler::CatalogHandler), true);
- webserver->RegisterUrlCallback("/catalog_object", "catalog_object.tmpl",
- MakeCallback(this, &ImpalaHttpHandler::CatalogObjectsHandler), false);
+ if(!FLAGS_use_local_catalog) {
+ // The /catalog_object endpoint is disabled if local_catalog_mode is used
+ // since metadata is partially fetched on demand
+ webserver->RegisterUrlCallback("/catalog_object", "catalog_object.tmpl",
+ MakeCallback(this, &ImpalaHttpHandler::CatalogObjectsHandler), false);
+ }
webserver->RegisterUrlCallback("/query_profile", "query_profile.tmpl",
MakeCallback(this, &ImpalaHttpHandler::QueryProfileHandler), false);
@@ -575,15 +580,21 @@ void ImpalaHttpHandler::CatalogHandler(const Webserver::WebRequest& req,
Value table_array(kArrayType);
for (const string& table: get_table_results.tables) {
Value table_obj(kObjectType);
- Value fq_name(Substitute("$0.$1", db.db_name, table).c_str(),
- document->GetAllocator());
- table_obj.AddMember("fqtn", fq_name, document->GetAllocator());
+ if(!FLAGS_use_local_catalog){
+ // Creates hyperlinks for /catalog_object. This is disabled in local catalog mode
+ Value fq_name(Substitute("$0.$1", db.db_name, table).c_str(),
+ document->GetAllocator());
+ table_obj.AddMember("fqtn", fq_name, document->GetAllocator());
+ }
Value table_name(table.c_str(), document->GetAllocator());
table_obj.AddMember("name", table_name, document->GetAllocator());
table_array.PushBack(table_obj, document->GetAllocator());
}
database.AddMember("num_tables", table_array.Size(), document->GetAllocator());
database.AddMember("tables", table_array, document->GetAllocator());
+ Value use_local_catalog(FLAGS_use_local_catalog);
+ database.AddMember("use_local_catalog", use_local_catalog,
+ document->GetAllocator());
databases.PushBack(database, document->GetAllocator());
}
document->AddMember("databases", databases, document->GetAllocator());
@@ -591,6 +602,7 @@ void ImpalaHttpHandler::CatalogHandler(const Webserver::WebRequest& req,
void ImpalaHttpHandler::CatalogObjectsHandler(const Webserver::WebRequest& req,
Document* document) {
+ DCHECK(!FLAGS_use_local_catalog);
const auto& args = req.parsed_args;
Webserver::ArgumentMap::const_iterator object_type_arg = args.find("object_type");
Webserver::ArgumentMap::const_iterator object_name_arg = args.find("object_name");
diff --git a/tests/custom_cluster/test_local_catalog.py b/tests/custom_cluster/test_local_catalog.py
index 65e93e0..e9bab64 100644
--- a/tests/custom_cluster/test_local_catalog.py
+++ b/tests/custom_cluster/test_local_catalog.py
@@ -381,6 +381,9 @@ class TestObservability(CustomClusterTestSuite):
impalad = self.cluster.impalads[0]
# Make sure local catalog mode is enabled and visible on web UI.
assert '(Local Catalog Mode)' in impalad.service.read_debug_webpage('/')
+ # Make sure /catalog_object endpoint is disabled on web UI.
+ assert 'No URI handler for '/catalog_object'' \
+ in impalad.service.read_debug_webpage('/catalog_object')
client = impalad.service.create_beeswax_client()
cache_hit_rate_metric_key = "catalog.cache.hit-rate"
cache_miss_rate_metric_key = "catalog.cache.miss-rate"
diff --git a/tests/webserver/test_web_pages.py b/tests/webserver/test_web_pages.py
index 9df9cb3..8deda42 100644
--- a/tests/webserver/test_web_pages.py
+++ b/tests/webserver/test_web_pages.py
@@ -314,22 +314,44 @@ class TestWebPage(ImpalaTestSuite):
def __test_catalog_object(self, db_name, tbl_name, cluster_properties):
"""Tests the /catalog_object endpoint for the given db/table. Runs
against an unloaded as well as a loaded table."""
+ obj_url = self.CATALOG_OBJECT_URL + \
+ "?object_type=TABLE&object_name={0}.{1}".format(db_name, tbl_name)
+
if cluster_properties.is_catalog_v2_cluster():
impalad_expected_str = \
- "UnsupportedOperationException: LocalCatalog.getTCatalogObject"
+ "No URI handler for '/catalog_object'"
+ self.client.execute("invalidate metadata %s.%s" % (db_name, tbl_name))
+ self.get_and_check_status(obj_url, tbl_name, ports_to_test=self.CATALOG_TEST_PORT)
+ # Catalog object endpoint is disabled in local catalog mode.
+ self.check_endpoint_is_disabled(obj_url, impalad_expected_str,
+ ports_to_test=self.IMPALAD_TEST_PORT)
+ self.client.execute("select count(*) from %s.%s" % (db_name, tbl_name))
+ self.get_and_check_status(obj_url, tbl_name, ports_to_test=self.CATALOG_TEST_PORT)
+ self.check_endpoint_is_disabled(obj_url, impalad_expected_str,
+ ports_to_test=self.IMPALAD_TEST_PORT)
else:
impalad_expected_str = tbl_name
- obj_url = self.CATALOG_OBJECT_URL + \
- "?object_type=TABLE&object_name={0}.{1}".format(db_name, tbl_name)
- self.client.execute("invalidate metadata %s.%s" % (db_name, tbl_name))
- self.get_and_check_status(obj_url, tbl_name, ports_to_test=self.CATALOG_TEST_PORT)
- self.get_and_check_status(obj_url, impalad_expected_str,
- ports_to_test=self.IMPALAD_TEST_PORT)
- self.client.execute("select count(*) from %s.%s" % (db_name, tbl_name))
-
- self.get_and_check_status(obj_url, tbl_name, ports_to_test=self.CATALOG_TEST_PORT)
- self.get_and_check_status(obj_url, impalad_expected_str,
- ports_to_test=self.IMPALAD_TEST_PORT)
+ self.client.execute("invalidate metadata %s.%s" % (db_name, tbl_name))
+ self.get_and_check_status(obj_url, tbl_name, ports_to_test=self.CATALOG_TEST_PORT)
+ self.get_and_check_status(obj_url, impalad_expected_str,
+ ports_to_test=self.IMPALAD_TEST_PORT)
+ self.client.execute("select count(*) from %s.%s" % (db_name, tbl_name))
+
+ self.get_and_check_status(obj_url, tbl_name, ports_to_test=self.CATALOG_TEST_PORT)
+ self.get_and_check_status(obj_url, impalad_expected_str,
+ ports_to_test=self.IMPALAD_TEST_PORT)
+
+ def check_endpoint_is_disabled(self, url, string_to_search="", ports_to_test=None):
+ """Helper method that verifies the given url does not exist."""
+ if ports_to_test is None:
+ ports_to_test = self.TEST_PORTS_WITH_SS
+ for port in ports_to_test:
+ input_url = url.format(port)
+ response = requests.get(input_url)
+ assert response.status_code == requests.codes.not_found, "URL: {0} Str:'{" \
+ "1}'\nResp:{2}".format(input_url, string_to_search, response.text)
+ assert string_to_search in response.text, "URL: {0} Str:'{1}'\nResp:{2}".format(
+ input_url, string_to_search, response.text)
def __test_table_metrics(self, db_name, tbl_name, metric):
self.client.execute("refresh %s.%s" % (db_name, tbl_name))
diff --git a/www/catalog.tmpl b/www/catalog.tmpl
index b6c99e6..20d9b75 100644
--- a/www/catalog.tmpl
+++ b/www/catalog.tmpl
@@ -39,7 +39,12 @@ under the License.
<tbody>
{{#large_tables}}
<tr>
- <td><a href="catalog_object?object_type=TABLE&object_name={{name}}">{{name}}</a>
+ <!-- The /catalog_object endpoint is disabled if local_catalog_mode is used
+ since metadata is partially fetched on demand. -->
+ {{?use_local_catalog}} {{name}} {{/use_local_catalog}}
+ {{^use_local_catalog}}
+ <a href="catalog_object?object_type=TABLE&object_name={{fqtn}}">{{name}}</a>
+ {{/use_local_catalog}}
</td>
<td>{{mem_estimate}}</td>
<td><a href="table_metrics?name={{name}}">{{name}}-metrics</a></td>
@@ -80,7 +85,11 @@ under the License.
<tbody>
{{#frequent_tables}}
<tr>
- <td><a href="catalog_object?object_type=TABLE&object_name={{name}}">{{name}}</a>
+ <td>
+ {{?use_local_catalog}} {{name}} {{/use_local_catalog}}
+ {{^use_local_catalog}}
+ <a href="catalog_object?object_type=TABLE&object_name={{fqtn}}">{{name}}</a>
+ {{/use_local_catalog}}
</td>
<td>{{num_metadata_ops}}</td>
<td><a href="table_metrics?name={{name}}">{{name}}-metrics</a></td>
@@ -120,7 +129,11 @@ under the License.
<tbody>
{{#high_file_count_tables}}
<tr>
- <td><a href="catalog_object?object_type=TABLE&object_name={{name}}">{{name}}</a>
+ <td>
+ {{?use_local_catalog}} {{name}} {{/use_local_catalog}}
+ {{^use_local_catalog}}
+ <a href="catalog_object?object_type=TABLE&object_name={{fqtn}}">{{name}}</a>
+ {{/use_local_catalog}}
</td>
<td>{{num_files}}</td>
<td><a href="table_metrics?name={{name}}">{{name}}-metrics</a></td>
@@ -152,10 +165,14 @@ under the License.
<div class="panel panel-info">
<div class="panel-heading">
+ {{^use_local_catalog}}
<a href='catalog_object?object_type=DATABASE&object_name={{name}}' id='{{name}}'>
+ {{/use_local_catalog}}
<h2 class="panel-title">{{name}}
<span class="pull-right">{{num_tables}} table(s)</span></h2>
+ {{^use_local_catalog}}
</a>
+ {{/use_local_catalog}}
</div>
<div class="panel-body">
<table id="{{name}}-tables" class='table table-hover table-bordered'>
@@ -170,7 +187,11 @@ under the License.
<tbody>
{{#tables}}
<tr>
- <td><a href="catalog_object?object_type=TABLE&object_name={{fqtn}}">{{name}}</a>
+ <td>
+ {{?use_local_catalog}} {{name}} {{/use_local_catalog}}
+ {{^use_local_catalog}}
+ <a href="catalog_object?object_type=TABLE&object_name={{fqtn}}">{{name}}</a>
+ {{/use_local_catalog}}
</td>
{{?has_metrics}}
<td><a href="table_metrics?name={{fqtn}}">{{name}}-metrics</a></td>