You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ar...@apache.org on 2019/04/11 18:25:12 UTC
[impala] branch master updated (45364a4 -> 10e052a)
This is an automated email from the ASF dual-hosted git repository.
arodoni pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git.
from 45364a4 IMPALA-8371: Return appropriate error code for unified backend tests
new 284a981 IMPALA-8385: Refactor Sentry admin user check
new 5eaa265 IMPALA-8227: Add support for WITH GRANT OPTION with Ranger
new 8f23bac IMPALA-8224: [DOCS] Add the missing contents about Impala web UI pages
new 65837f8 IMPALA-8391: [DOCS] ALTER TABLE RENAME now renames the underlying Kudu TABLE
new 50e259e Bump CDH_BUILD_NUMBER to 1009254
new 10e052a IMPALA-7981: Add host disk statistics to profile
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/catalog/catalog-server.cc | 2 +-
be/src/catalog/catalog.cc | 7 +-
be/src/catalog/catalog.h | 7 +-
be/src/exec/catalog-op-executor.cc | 8 +-
be/src/exec/catalog-op-executor.h | 9 +-
be/src/runtime/query-state.cc | 9 +
be/src/service/client-request-state.cc | 24 -
be/src/service/fe-support.cc | 29 +
be/src/util/disk-info.h | 6 +
be/src/util/system-state-info-test.cc | 51 +
be/src/util/system-state-info.cc | 175 ++--
be/src/util/system-state-info.h | 82 +-
bin/impala-config.sh | 2 +-
common/thrift/CatalogService.thrift | 6 +-
docs/topics/impala_metadata.xml | 8 +-
docs/topics/impala_tables.xml | 59 +-
docs/topics/impala_webui.xml | 1070 +++++++++++++++-----
.../impala/authorization/AuthorizationManager.java | 6 -
.../authorization/NoneAuthorizationFactory.java | 5 -
.../ranger/RangerCatalogdAuthorizationManager.java | 15 +-
.../sentry/SentryCatalogdAuthorizationManager.java | 6 +-
.../sentry/SentryImpaladAuthorizationManager.java | 55 +-
.../java/org/apache/impala/service/FeSupport.java | 11 +
.../java/org/apache/impala/service/Frontend.java | 12 +-
.../java/org/apache/impala/service/JniCatalog.java | 24 +-
tests/authorization/test_ranger.py | 93 +-
tests/authorization/test_sentry.py | 60 ++
tests/query_test/test_observability.py | 7 +-
28 files changed, 1386 insertions(+), 462 deletions(-)
create mode 100644 tests/authorization/test_sentry.py
[impala] 02/06: IMPALA-8227: Add support for WITH GRANT OPTION with
Ranger
Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
arodoni pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 5eaa265f6737633b4de1bdd68731d387780d4a76
Author: Austin Nobis <an...@cloudera.com>
AuthorDate: Thu Apr 4 08:56:52 2019 -0700
IMPALA-8227: Add support for WITH GRANT OPTION with Ranger
This patch adds support for the WITH GRANT OPTION SQL syntax when
granting privileges to users and groups in Ranger. This allows users who
have been granted a privilege to then grant that privilege to other
users/groups.
Testing:
- Ran all FE tests
- Ran authorization E2E tests
- Added an E2E authorization test in test_ranger to verify the
functionality
Change-Id: I9c2384f0a9fe30bea1eaceac5b27b1c432383aa8
Reviewed-on: http://gerrit.cloudera.org:8080/12962
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../ranger/RangerCatalogdAuthorizationManager.java | 10 +--
tests/authorization/test_ranger.py | 93 +++++++++++++++++++++-
2 files changed, 97 insertions(+), 6 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/authorization/ranger/RangerCatalogdAuthorizationManager.java b/fe/src/main/java/org/apache/impala/authorization/ranger/RangerCatalogdAuthorizationManager.java
index 22c4b04..ca768b8 100644
--- a/fe/src/main/java/org/apache/impala/authorization/ranger/RangerCatalogdAuthorizationManager.java
+++ b/fe/src/main/java/org/apache/impala/authorization/ranger/RangerCatalogdAuthorizationManager.java
@@ -190,8 +190,8 @@ public class RangerCatalogdAuthorizationManager implements AuthorizationManager
for (TPrivilege p: privileges) {
Function<Map<String, String>, GrantRevokeRequest> createRequest = (resource) ->
- createGrantRevokeRequest(grantor, user, groups, clusterName, p.privilege_level,
- resource);
+ createGrantRevokeRequest(grantor, user, groups, clusterName, p.has_grant_opt,
+ p.privilege_level, resource);
// Ranger Impala service definition defines 3 resources:
// [DB -> Table -> Column]
@@ -220,13 +220,13 @@ public class RangerCatalogdAuthorizationManager implements AuthorizationManager
}
private static GrantRevokeRequest createGrantRevokeRequest(String grantor, String user,
- List<String> groups, String clusterName, TPrivilegeLevel level,
- Map<String, String> resource) {
+ List<String> groups, String clusterName, boolean withGrantOpt,
+ TPrivilegeLevel level, Map<String, String> resource) {
GrantRevokeRequest request = new GrantRevokeRequest();
request.setGrantor(grantor);
if (user != null) request.getUsers().add(user);
if (!groups.isEmpty()) request.getGroups().addAll(groups);
- request.setDelegateAdmin(Boolean.FALSE);
+ request.setDelegateAdmin((withGrantOpt) ? Boolean.TRUE : Boolean.FALSE);
request.setEnableAudit(Boolean.TRUE);
request.setReplaceExistingPermissions(Boolean.FALSE);
request.setClusterName(clusterName);
diff --git a/tests/authorization/test_ranger.py b/tests/authorization/test_ranger.py
index ab70cd5..a8ebb90 100644
--- a/tests/authorization/test_ranger.py
+++ b/tests/authorization/test_ranger.py
@@ -18,12 +18,17 @@
# Client tests for SQL statement authorization
import grp
+import json
import pytest
+import requests
import time
-from getpass import getuser
+from getpass import getuser
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
+RANGER_AUTH = ("admin", "admin")
+RANGER_HOST = "http://localhost:6080"
+
class TestRanger(CustomClusterTestSuite):
"""
@@ -81,3 +86,89 @@ class TestRanger(CustomClusterTestSuite):
.format(unique_database, kw, ident), user=admin)
admin_client.execute("drop database if exists {0} cascade"
.format(unique_database), user=admin)
+
+ @CustomClusterTestSuite.with_args(
+ impalad_args="--server-name=server1 --ranger_service_type=hive "
+ "--ranger_app_id=impala "
+ "--authorization_factory_class="
+ "org.apache.impala.authorization.ranger.RangerAuthorizationFactory",
+ catalogd_args="--server-name=server1 --ranger_service_type=hive "
+ "--ranger_app_id=impala "
+ "--authorization_factory_class="
+ "org.apache.impala.authorization.ranger.RangerAuthorizationFactory")
+ def test_grant_option(self, unique_name):
+ user1 = getuser()
+ user2 = unique_name + "_user"
+ admin = "admin"
+ admin_client = self.create_impala_client()
+ user1_client = self.create_impala_client()
+ user2_client = self.create_impala_client()
+ unique_database = unique_name + "_db"
+ unique_table = unique_name + "_tbl"
+ id = self._add_ranger_user(user2)
+
+ try:
+ # Set-up temp database/table
+ admin_client.execute("drop database if exists {0} cascade".format(unique_database),
+ user=admin)
+ admin_client.execute("create database {0}".format(unique_database), user=admin)
+ admin_client.execute("create table {0}.{1} (x int)"
+ .format(unique_database, unique_table), user=admin)
+
+ # Give user 1 the ability to grant select privileges on unique_database
+ self.execute_query_expect_success(admin_client,
+ "grant select on database {0} to user {1} with "
+ "grant option".format(unique_database, user1),
+ user=admin)
+ # TODO: IMPALA-8293 use refresh authorization
+ time.sleep(10)
+ # User 1 grants select privilege to user 2
+ self.execute_query_expect_success(user1_client,
+ "grant select on database {0} to user {1}"
+ .format(unique_database, user2), user=user1)
+ # TODO: IMPALA-8293 use refresh authorization
+ time.sleep(10)
+ # User 2 exercises select privilege
+ self.execute_query_expect_success(user2_client, "show tables in {0}"
+ .format(unique_database), user=user2)
+ # User 1 revokes select privilege from user 2
+ self.execute_query_expect_success(user1_client,
+ "revoke select on database {0} from user "
+ "{1}".format(unique_database, user2), user=user1)
+ # TODO: IMPALA-8293 use refresh authorization
+ time.sleep(10)
+ # User 2 can no longer select because the privilege was revoked
+ self.execute_query_expect_failure(user2_client, "show tables in {0}"
+ .format(unique_database))
+ # Revoke privilege granting from user 1
+ self.execute_query_expect_success(admin_client, "revoke grant option for select "
+ "on database {0} from user {1}"
+ .format(unique_database, user1), user=admin)
+ # TODO: IMPALA-8293 use refresh authorization
+ time.sleep(10)
+ # User 1 can no longer grant privileges on unique_database
+ self.execute_query_expect_failure(user1_client,
+ "grant select on database {0} to user {1}"
+ .format(unique_database, user2), user=user1)
+ finally:
+ admin_client.execute("revoke select on database {0} from user {1}"
+ .format(unique_database, user2), user=admin)
+ admin_client.execute("revoke grant option for select on database {0} from user {1}"
+ .format(unique_database, user1), user=admin)
+ admin_client.execute("drop database if exists {0} cascade".format(unique_database),
+ user=admin)
+ self._remove_ranger_user(id)
+
+ def _add_ranger_user(self, user):
+ data = {"name": user, "password": "password123", "userRoleList": ["ROLE_USER"]}
+ headers = {"Content-Type": "application/json", "Accept": "application/json"}
+
+ r = requests.post("{0}/service/xusers/secure/users".format(RANGER_HOST),
+ auth=RANGER_AUTH,
+ json=data, headers=headers)
+ return json.loads(r.content)['id']
+
+ def _remove_ranger_user(self, id):
+ r = requests.delete("{0}/service/xusers/users/{1}?forceDelete=true"
+ .format(RANGER_HOST, id), auth=RANGER_AUTH)
+ assert r.status_code < 300 and r.status_code >= 200
[impala] 05/06: Bump CDH_BUILD_NUMBER to 1009254
Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
arodoni pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 50e259e3691b7de36f58408bde6413880c74845e
Author: Bharath Krishna <bh...@cloudera.com>
AuthorDate: Wed Apr 10 08:40:20 2019 -0700
Bump CDH_BUILD_NUMBER to 1009254
This change bumps the CDH_BUILD_VERSION to a version that includes
the change to have full-thrift object for DropDatabaseMessage in
Metastore notifications, added in HIVE-21526. This change is needed
for the upcoming patch for IMPALA-8338.
Testing:
- Ran a full exhaustive build using the impala-private-parameterized job.
Change-Id: Id38bae921c4d93421c6c72cdccef6a4783e2588e
Reviewed-on: http://gerrit.cloudera.org:8080/12965
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
bin/impala-config.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 05e4ff0..eae7c06 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -160,7 +160,7 @@ fi
: ${CDH_DOWNLOAD_HOST:=native-toolchain.s3.amazonaws.com}
export CDH_DOWNLOAD_HOST
export CDH_MAJOR_VERSION=6
-export CDH_BUILD_NUMBER=909265
+export CDH_BUILD_NUMBER=1009254
export CDP_BUILD_NUMBER=976603
export IMPALA_HADOOP_VERSION=3.0.0-cdh6.x-SNAPSHOT
export IMPALA_HBASE_VERSION=2.1.0-cdh6.x-SNAPSHOT
[impala] 03/06: IMPALA-8224: [DOCS] Add the missing contents about
Impala web UI pages
Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
arodoni pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 8f23bac91ae8f1b98ccb7728a02b0179265f5ef8
Author: Alex Rodoni <ar...@cloudera.com>
AuthorDate: Thu Mar 28 16:29:28 2019 -0700
IMPALA-8224: [DOCS] Add the missing contents about Impala web UI pages
Change-Id: Ic30c26fe1d19c1a81704723cd5ebed667982d5b6
Reviewed-on: http://gerrit.cloudera.org:8080/12934
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Bikramjeet Vig <bi...@cloudera.com>
---
docs/topics/impala_metadata.xml | 8 +-
docs/topics/impala_webui.xml | 1070 +++++++++++++++++++++++++++++----------
2 files changed, 796 insertions(+), 282 deletions(-)
diff --git a/docs/topics/impala_metadata.xml b/docs/topics/impala_metadata.xml
index 02fcf55..f35a2f0 100644
--- a/docs/topics/impala_metadata.xml
+++ b/docs/topics/impala_metadata.xml
@@ -298,6 +298,7 @@ under the License.
<concept id="event_processor_metrics">
<title>Metrics for Event Based Automatic Metadata Sync</title>
+ <!--AR 3/29/2019 Incorporate this section to the web UI doc when this feature goes GA.-->
<conbody>
@@ -307,13 +308,6 @@ under the License.
</p>
<p>
- By default, the debug web UI of <codeph>catalogd</codeph> is at
- <codeph>http://<varname>impala-server-hostname</varname>:25020</codeph> (non-secure
- cluster) or <codeph>https://<varname>impala-server-hostname</varname>:25020</codeph>
- (secure cluster).
- </p>
-
- <p>
Under the web UI, there are two pages that presents the metrics for HMS event
processor that is responsible for the event based automatic metadata sync.
<ul>
diff --git a/docs/topics/impala_webui.xml b/docs/topics/impala_webui.xml
index 3a5d187..966f6f8 100644
--- a/docs/topics/impala_webui.xml
+++ b/docs/topics/impala_webui.xml
@@ -21,7 +21,13 @@ under the License.
<concept id="webui">
<title id="webui_title">Impala Web User Interface for Debugging</title>
- <titlealts audience="PDF"><navtitle>Web User Interface</navtitle></titlealts>
+
+ <titlealts audience="PDF">
+
+ <navtitle>Web User Interface</navtitle>
+
+ </titlealts>
+
<prolog>
<metadata>
<data name="Category" value="Impala"/>
@@ -34,55 +40,77 @@ under the License.
<conbody>
- <p id="webui_intro">Each of the Impala daemons (<cmdname>impalad</cmdname>,
- <cmdname>statestored</cmdname>, and <cmdname>catalogd</cmdname>)
- includes a built-in web server that displays diagnostic and status
- information: <ul>
- <li>
- <p> The <cmdname>impalad</cmdname> web UI (default port: 25000)
- includes information about configuration settings, running and
- completed queries, and associated performance and resource usage for
- queries. In particular, the <uicontrol>Details</uicontrol> link for
- each query displays alternative views of the query including a
- graphical representation of the plan, and the output of the
- <codeph>EXPLAIN</codeph>, <codeph>SUMMARY</codeph>, and
- <codeph>PROFILE</codeph> statements from
- <cmdname>impala-shell</cmdname>. Each host that runs the
- <cmdname>impalad</cmdname> daemon has its own instance of the web
- UI, with details about those queries for which that host served as
- the coordinator. The <cmdname>impalad</cmdname> web UI is mainly for
- diagnosing query problems that can be traced to a particular node.
- </p>
- </li>
- <li>
- <p> The <cmdname>statestored</cmdname> web UI (default port: 25010)
- includes information about memory usage, configuration settings, and
- ongoing health checks performed by this daemon. Because there is
- only a single instance of this daemon within any cluster, you view
- the web UI only on the particular host that serves as the Impala
- StateStore. </p>
- </li>
- <li>
- <p> The <cmdname>catalogd</cmdname> web UI (default port: 25020)
- includes information about the databases, tables, and other objects
- managed by Impala, in addition to the resource usage and
- configuration settings of the daemon itself. The catalog information
- is represented as the underlying Thrift data structures. Because
- there is only a single instance of this daemon within any cluster,
- you view the web UI only on the particular host that serves as the
- Impala Catalog Server. </p>
- </li>
- </ul>
+ <p id="webui_intro">
+ Each of the Impala daemons (<cmdname>impalad</cmdname>, <cmdname>statestored</cmdname>,
+ and <cmdname>catalogd</cmdname>) includes a built-in web server that displays diagnostic
+ and status information.
</p>
- <note>
- <p>
- The web user interface is primarily for problem diagnosis and troubleshooting. The items listed and their
- formats are subject to change.
- </p>
- </note>
+ <dl>
+ <dlentry>
+
+ <dt>
+ impalad Web UI
+ </dt>
+
+ <dd>
+ <p>
+ The <cmdname>impalad</cmdname> Web UI includes information about configuration
+ settings, running and completed queries, and associated performance and resource
+ usage for queries. In particular, the <uicontrol>Details</uicontrol> link for each
+ query displays alternative views of the query including a graphical representation
+ of the plan, and the output of the <codeph>EXPLAIN</codeph>,
+ <codeph>SUMMARY</codeph>, and <codeph>PROFILE</codeph> statements from
+ <cmdname>impala-shell</cmdname>. Each host that runs the <cmdname>impalad</cmdname>
+ daemon has its own instance of the Web UI, with details about those queries for
+ which that host served as the coordinator. The <cmdname>impalad</cmdname> Web UI is
+ primarily used for diagnosing query problems that can be traced to a particular
+ node.
+ </p>
+ </dd>
+
+ </dlentry>
+
+ <dlentry>
+
+ <dt>
+ statestored Web UI
+ </dt>
+
+ <dd>
+ <p>
+ The <cmdname>statestored</cmdname> Web UI includes information about memory usage,
+ configuration settings, and ongoing health checks performed by
+ <codeph>statestored</codeph>. Because there is only a single instance of the
+ <codeph>statestored</codeph> within any Impala cluster, you access the Web UI only
+ on the particular host that serves as the Impala StateStore.
+ </p>
+ </dd>
+
+ </dlentry>
+
+ <dlentry>
+
+ <dt>
+ catalogd Web UI
+ </dt>
+
+ <dd>
+ <p>
+ The <cmdname>catalogd</cmdname> Web UI includes information about the databases,
+ tables, and other objects managed by Impala, in addition to the resource usage and
+ configuration settings of the <codeph>catalogd</codeph>. Because there is only a
+ single instance of the <codeph>catalogd</codeph> within any Impala cluster, you
+ access the Web UI only on the particular host that serves as the Impala Catalog
+ Server.
+ </p>
+ </dd>
+
+ </dlentry>
+ </dl>
<p outputclass="toc inpage"/>
+
</conbody>
<concept id="webui_impalad">
@@ -92,23 +120,43 @@ under the License.
<conbody>
<p>
- To debug and troubleshoot the <cmdname>impalad</cmdname> daemon using a web-based interface, open the URL
- <codeph>http://<varname>impala-server-hostname</varname>:25000/</codeph> in a browser. (For secure
- clusters, use the prefix <codeph>https://</codeph> instead of <codeph>http://</codeph>.) Because each
- Impala node produces its own set of debug information, choose a specific node that you are curious about or
- suspect is having problems.
+ To debug and troubleshoot an <cmdname>impalad</cmdname> using a web-based interface,
+ open the URL
+ <codeph>http://<varname>impala‑server‑hostname</varname>:25000/</codeph> in
+ a browser. (For secure clusters, use the prefix <codeph>https://</codeph> instead of
+ <codeph>http://</codeph>.)
+ </p>
+
+ <p>
+ Because each Impala node produces its own set of debug information, you should choose a
+ specific node that you want to investigate an issue on.
</p>
</conbody>
- <concept audience="hidden" id="webui_impalad_disabling">
+ <concept id="webui_impalad_disabling">
<title>Turning off the Web UI for impalad</title>
<conbody>
- <p>To disable web UI for an impalad, restart </p>
+ <p>
+ To disable Web UI for an <codeph>impalad</codeph>:
+ </p>
+
+ <ol>
+ <li>
+ Stop the <codeph>impalad</codeph>.
+ </li>
+
+ <li>
+ Restart the <codeph>impalad</codeph> with the
+ <codeph>‑‑enable_webserver=false</codeph> startup flag.
+ </li>
+ </ol>
+
</conbody>
+
</concept>
<concept id="webui_impalad_main">
@@ -118,90 +166,208 @@ under the License.
<conbody>
<p>
- By default, the main page of the debug web UI is at
- <codeph>http://<varname>impala-server-hostname</varname>:25000/</codeph> (non-secure cluster) or
- <codeph>https://<varname>impala-server-hostname</varname>:25000/</codeph> (secure cluster).
+ The main <codeph>impalad</codeph> Web UI page at <b>/</b> lists the following
+ information about the <codeph>impalad</codeph>:
</p>
- <p>
- This page lists the version of the <cmdname>impalad</cmdname> daemon, plus basic hardware and software
- information about the corresponding host, such as information about the CPU, memory, disks, and operating
- system version.
- </p>
+ <ul>
+ <li>
+ The version of the <cmdname>impalad</cmdname> daemon
+ <ul>
+ <li>
+ The "Version" section contains some other information too, like when Impala was
+ built, what build flags were used.
+ </li>
+ </ul>
+ </li>
+
+ <li>
+ Process start time
+ </li>
+
+ <li>
+ Hardware information
+ </li>
+
+ <li>
+ OS information
+ </li>
+
+ <li>
+ Process information
+ </li>
+
+ <li>
+ CGroup information
+ </li>
+ </ul>
+
</conbody>
+
</concept>
+
<concept id="webui_impalad_admission">
- <title>Admission Control Page</title>
- <conbody>
- <p> By default, the <uicontrol>admission</uicontrol> page of the debug
- web UI is at
- <codeph>http://<varname>impala-server-hostname</varname>:25000/admission</codeph>
- (non-secure cluster) or
- <codeph>https://<varname>impala-server-hostname</varname>:25000/admission</codeph>
- (secure cluster). </p>
- <p>Use the <uicontrol>admission</uicontrol> page to troubleshoot queued
- queries and the admission control.</p>
- <p>The <uicontrol>admission</uicontrol> page provides the following
- information about each resource pool to which queries have been
- submitted at least once:</p>
+
+ <title>Admission Controller Page</title>
+
+ <conbody>
+
+ <p>
+ The <b>Admission Controller</b> <codeph>impalad</codeph> debug Web UI is at
+ <uicontrol>/admission</uicontrol> page under the main <codeph>impalad</codeph> Web UI.
+ </p>
+
+ <p>
+ Use the <uicontrol>/admission</uicontrol> page to troubleshoot queued queries and the
+ admission control.
+ </p>
+
+ <p>
+ The <uicontrol>admission</uicontrol> page provides the following information about
+ each resource pool to which queries have been submitted at least once:
+ </p>
+
<ul>
- <li>Time since the <codeph>statestored</codeph> received the last
- update</li>
- <li>A warning if this <codeph>impalad</codeph> is considered
- disconnected from the <codeph>statestored</codeph> and thus the
- information on this page could be stale.</li>
- <li>Pool configuration</li>
- <li>Queued queries submitted to this coordinator, in the order of
- submission</li>
- <li>Running queries submitted to this coordinator</li>
- <li>Pool stats<ul>
- <li>Average of time in queue<p>This is an exponential moving
- average with the factor of 0.2, which considers approximately
- past 10 queries. If a query is admitted immediately, the wait
- time of 0 is used in calculating this average wait
- time.</p></li>
- </ul></li>
- <li>Histogram of the distribution of peak memory used by queries
- admitted to the pool<p>Use the histogram to figure out the minimum
- and maximum query <codeph>MEM_LIMIT</codeph> ranges of the this
- pool.</p><p>The histogram displays data for all queries admitted
- to the pool, including the queries finished, got canceled, or hit
- an error. </p></li>
+ <li>
+ Time since the <codeph>statestored</codeph> received the last update
+ </li>
+
+ <li>
+ A warning if this <codeph>impalad</codeph> is considered disconnected from the
+ <codeph>statestored</codeph> and thus the information on this page could be stale.
+ </li>
+
+ <li>
+ Pool configuration
+ </li>
+
+ <li>
+ Queued queries submitted to this coordinator, in the order of submission
+ </li>
+
+ <li>
+ Running queries submitted to this coordinator
+ </li>
+
+ <li>
+ Pool stats
+ <ul>
+ <li>
+ Average of time in queue
+ <p>
+ This is an exponential moving average which represents the average time in
+ queue over the last 10 to 12 queries. If a query is admitted immediately, the
+ wait time of 0 is used in calculating this average wait time.
+ </p>
+ </li>
+ </ul>
+ </li>
+
+ <li>
+ Histogram of the distribution of peak memory used by queries admitted to the pool
+ <p>
+ Use the histogram to figure out settings for the minimum and maximum query
+ <codeph>MEM_LIMIT</codeph> ranges for this pool.
+ </p>
+
+ <p>
+ The histogram displays data for all queries admitted to the pool, including the
+ queries that finished, got canceled, or hit an error.
+ </p>
+ </li>
</ul>
- <p>Click on the pool name to only display information relevant to that
- pool. You can then refresh the debug page to see only the information
- for that specific pool.</p>
- <p>You can reset the stats that keep track of historical data, such as
- <b>Totals</b> stats, <b>Time in queue (exponential moving
- average)</b>, and the histogram.</p>
- <p>The above information is also available as a JSON object from the
- following HTTP
- endpoint:<codeblock><codeph>http://<varname>impala-server-hostname</varname>:<varname>port</varname>/admission?json</codeph></codeblock></p>
- <p>See <xref href="impala_admission.xml#admission_control"/> for the
- description of the properties in admission control.</p>
+
+ <p>
+ Click on the pool name to only display information relevant to that pool. You can then
+ refresh the debug page to see only the information for that specific pool.
+ </p>
+
+ <p>
+ Click <b>Reset informational stats for all pools</b> to reset the stats that keep
+ track of historical data, such as <b>Totals</b> stats, <b>Time in queue (exponential
+ moving average)</b>, and the histogram.
+ </p>
+
+ <p>
+ The above information is also available as a JSON object from the following HTTP
+ endpoint:
+<codeblock><codeph>http://<varname>impala-server-hostname</varname>:<varname>port</varname>/admission?json</codeph></codeblock>
+ </p>
+
+ <p>
+ See <xref href="impala_admission.xml#admission_control"/> for the description of the
+ properties in admission control.
+ </p>
+
</conbody>
+
</concept>
<concept id="webui_impalad_backends">
- <title>Backends Page</title>
+ <title>Known Backends Page</title>
<conbody>
<p>
- By default, the <uicontrol>backends</uicontrol> page of the debug web UI is at
- <codeph>http://<varname>impala-server-hostname</varname>:25000/backends</codeph> (non-secure cluster) or
- <codeph>https://<varname>impala-server-hostname</varname>:25000/backends</codeph> (secure cluster).
+ The <b>Known backends</b> page of the <codeph>impalad</codeph> debug Web UI is at
+ <b>/backends</b> under the main <codeph>impalad</codeph> Web UI.
</p>
- <p> This page lists the host and port info for each of the
- <cmdname>impalad</cmdname> nodes in the cluster. Because each
- <cmdname>impalad</cmdname> daemon knows about every other
- <cmdname>impalad</cmdname> daemon through the StateStore, this
- information should be the same regardless of which node you select.
- Links take you to the corresponding debug web pages for any of the
- other nodes in the cluster. </p>
+ <p>
+ This page lists the following info for each of the <cmdname>impalad</cmdname> nodes in
+ the cluster. Because each <cmdname>impalad</cmdname> daemon knows about every other
+ <cmdname>impalad</cmdname> daemon through the StateStore, this information should be
+ the same regardless of which node you select.
+ </p>
+
+ <ul>
+ <li>
+ Address of the node: Host name and port
+ </li>
+
+ <li>
+ KRPC address: The KRPC address of the <codeph>impalad</codeph>. Use this address
+ when you issue the <codeph>SHUTDOWN</codeph> command for this
+ <codeph>impalad</codeph>.
+ </li>
+
+ <li>
+ Whether acting as a coordinator
+ </li>
+
+ <li>
+ Whether acting as an executor
+ </li>
+
+ <li>
+ Quiescing status: Specify whether the graceful shutdown process has been initiated
+ on this <codeph>impalad</codeph>.
+ </li>
+
+ <li>
+ Memory limit for admission: The amount of memory that can be admitted to this
+ backend by the admission controller.
+ </li>
+
+ <li>
+ Memory reserved: The amount of memory reserved by queries that are active, either
+ currently executing or finished but not yet closed, on this backend.
+ <p>
+ The memory reserved for a query that is currently executing is its memory limit,
+ if set. Otherwise, if the query has no limit or if the query finished executing,
+ the current consumption is used.
+ </p>
+ </li>
+
+ <li>
+ Memory of the queries admitted to this coordinator: The memory submitted to this
+ particular host by the queries admitted by this coordinator.
+ </li>
+ </ul>
+
</conbody>
+
</concept>
<concept id="webui_impalad_catalog">
@@ -211,46 +377,105 @@ under the License.
<conbody>
<p>
- By default, the <uicontrol>catalog</uicontrol> page of the debug web UI is at
- <codeph>http://<varname>impala-server-hostname</varname>:25000/catalog</codeph> (non-secure cluster) or
- <codeph>https://<varname>impala-server-hostname</varname>:25000/catalog</codeph> (secure cluster).
+ The <b>Catalog</b> page of the <codeph>impalad</codeph> debug Web UI is at
+ <b>/catalog</b> under the main <codeph>impalad</codeph> Web UI.
</p>
<p>
- This page displays a list of databases and associated tables recognized by this instance of
- <cmdname>impalad</cmdname>. You can use this page to locate which database a table is in, check the
- exact spelling of a database or table name, look for identical table names in multiple databases, and so
- on.
+ This page displays a list of databases and associated tables recognized by this
+ instance of <cmdname>impalad</cmdname>. You can use this page to locate which database
+ a table is in, check the exact spelling of a database or table name, look for
+ identical table names in multiple databases. The primary debugging use case would be
+ to check if an <codeph>impalad</codeph> instance has knowledge of a particular table
+ that someone expects to be in a particular database.
</p>
+
</conbody>
+
+ </concept>
+
+ <concept id="webui_impalad_hadoop-varz">
+
+ <title>Hadoop Configuration</title>
+
+ <conbody>
+
+ <p>
+ The <b>Hadoop Configuration</b> page of the <codeph>impalad</codeph> debug Web UI is
+ at <b>/hadoop-varz</b> under the main <codeph>impalad</codeph> Web UI.
+ </p>
+
+ <p>
+ This page displays the Hadoop common configurations that Impala is running with.
+ </p>
+
+ </conbody>
+
+ </concept>
+
+ <concept id="webui_impalad_jmx">
+
+ <title>JMX</title>
+
+ <conbody>
+
+ <p>
+ The <b>JMX</b> page of the <codeph>impalad</codeph> debug Web UI is at <b>/jmx</b>
+ under the main <codeph>impalad</codeph> Web UI.
+ </p>
+
+ <p>
+ This page displays monitoring information about various JVM subsystems, such as memory
+ pools, thread management, runtime. etc.
+ </p>
+
+ </conbody>
+
+ </concept>
+
+ <concept id="webui_impalad_log_level">
+
+ <title>Java Log Level</title>
+
+ <conbody>
+
+ <p>
+ The <b>Change log level</b> page of the <codeph>impalad</codeph> debug Web UI is at
+ <b>/log_level</b> under the main <codeph>impalad</codeph> Web UI.
+ </p>
+
+ <p>
+ This page displays the current Java and backend log levels, and it allows you to
+ change the log levels dynamically without having to restart the
+ <codeph>impalad</codeph>.
+ </p>
+
+ </conbody>
+
</concept>
<concept id="webui_impalad_logs">
<title>Logs Page</title>
- <prolog>
- <metadata>
- <data name="Category" value="Logs"/>
- </metadata>
- </prolog>
<conbody>
<p>
- By default, the <uicontrol>logs</uicontrol> page of the debug web UI is at
- <codeph>http://<varname>impala-server-hostname</varname>:25000/logs</codeph> (non-secure cluster) or
- <codeph>https://<varname>impala-server-hostname</varname>:25000/logs</codeph> (secure cluster).
+ The <b>INFO logs</b> page of the <codeph>impalad</codeph> debug Web UI is at
+ <b>/logs</b> under the main <codeph>impalad</codeph> Web UI.
</p>
<p>
- This page shows the last portion of the <filepath>impalad.INFO</filepath> log file, the most detailed of
- the info, warning, and error logs for the <cmdname>impalad</cmdname> daemon. You can refer here to see
- the details of the most recent operations, whether the operations succeeded or encountered errors. This
- central page can be more convenient than looking around the filesystem for the log files, which could be
- in different locations on clusters that use cluster management software.
+ This page shows the last portion of the <codeph>impalad.INFO</codeph> log file,
+ including the info, warning, and error logs for the <cmdname>impalad</cmdname>. You
+ can see the details of the most recent operations, whether the operations succeeded or
+ encountered errors. This page provides one central place for the log files and saves
+ you from looking around the filesystem for the log files, which could be in different
+ locations on clusters that use cluster management software.
</p>
</conbody>
+
</concept>
<concept id="webui_impalad_memz">
@@ -260,17 +485,17 @@ under the License.
<conbody>
<p>
- By default, the <uicontrol>memz</uicontrol> page of the debug web UI is at
- <codeph>http://<varname>impala-server-hostname</varname>:25000/memz</codeph> (non-secure cluster) or
- <codeph>https://<varname>impala-server-hostname</varname>:25000/memz</codeph> (secure cluster).
+ The <b>Memory Usage</b> page of the <codeph>impalad</codeph> debug Web UI is at
+ <b>/memz</b> under the main <codeph>impalad</codeph> Web UI.
</p>
<p>
- This page displays summary and detailed information about memory usage by the <cmdname>impalad</cmdname>
- daemon. You can see the memory limit in effect for the node, and how much of that memory Impala is
- currently using.
+ This page displays the summary and detailed information about memory usage by the
+ <codeph>impalad</codeph>.
</p>
+
</conbody>
+
</concept>
<concept id="webui_impalad_metrics">
@@ -280,17 +505,17 @@ under the License.
<conbody>
<p>
- By default, the <uicontrol>metrics</uicontrol> page of the debug web UI is at
- <codeph>http://<varname>impala-server-hostname</varname>:25000/metrics</codeph> (non-secure cluster) or
- <codeph>https://<varname>impala-server-hostname</varname>:25000/metrics</codeph> (secure cluster).
+ The <b>Metrics</b> page of the <codeph>impalad</codeph> debug Web UI is at
+ <b>/metrics</b> under the main <codeph>impalad</codeph> Web UI.
</p>
<p>
- This page displays the current set of metrics: counters and flags representing various aspects of
- <cmdname>impalad</cmdname> internal operation.
+ This page displays the current set of metrics, counters and flags representing various
+ aspects of <codeph>impalad</codeph> internal operations.
</p>
</conbody>
+
</concept>
<concept id="webui_impalad_queries">
@@ -300,30 +525,83 @@ under the License.
<conbody>
<p>
- By default, the <uicontrol>queries</uicontrol> page of the debug web UI is at
- <codeph>http://<varname>impala-server-hostname</varname>:25000/queries</codeph> (non-secure cluster) or
- <codeph>https://<varname>impala-server-hostname</varname>:25000/queries</codeph> (secure cluster).
+ The <b>Queries</b> page of the <codeph>impalad</codeph> debug Web UI is at
+ <b>/queries</b> under the main <codeph>impalad</codeph> Web UI.
</p>
<p>
- This page lists all currently running queries, plus any completed queries whose details still reside in
- memory. The queries are listed in reverse chronological order, with the most recent at the top. (You can
- control the amount of memory devoted to completed queries by specifying the
- <codeph>--query_log_size</codeph> startup option for <cmdname>impalad</cmdname>.)
+ This page lists:
+ <ul>
+ <li>
+ Currently running queries
+ </li>
+
+ <li>
+ Queries that have completed their execution, but have not been closed yet
+ </li>
+
+ <li>
+ Completed queries whose details still reside in memory
+ </li>
+ </ul>
+ The queries are listed in reverse chronological order, with the most recent at the
+ top. You can control the amount of memory devoted to completed queries by specifying
+ the <codeph>-‑-‑query_log_size</codeph> startup option for
+ <codeph>impalad</codeph>.
</p>
<p>
- On this page, you can see at a glance how many SQL statements are failing (<codeph>State</codeph> value
- of <codeph>EXCEPTION</codeph>), how large the result sets are (<codeph># rows fetched</codeph>), and how
- long each statement took (<codeph>Start Time</codeph> and <codeph>End Time</codeph>).
+ This page provides:
+ <ul>
+ <li>
+ How many SQL statements are failing (<codeph>State</codeph> value of
+ <codeph>EXCEPTION</codeph>)
+ </li>
+
+ <li>
+ How large the result sets are (<codeph># rows fetched</codeph>)
+ </li>
+
+ <li>
+ How long each statement took (<codeph>Start Time</codeph> and <codeph>End
+ Time</codeph>)
+ </li>
+ </ul>
</p>
<p>
- Each query has an associated link that displays the detailed query profile, which you can examine to
- understand the performance characteristics of that query. See
+ Each query has an associated link that displays the detailed query profile, which you
+ can examine to understand the performance characteristics of that query. See
<xref href="impala_explain_plan.xml#perf_profile"/> for details.
</p>
+
+ <p>
+ The <b>Queries</b> page also includes the <b>Query Locations</b> section that lists
+ the number of running queries with fragments on this host.
+ </p>
+
</conbody>
+
+ </concept>
+
+ <concept id="webui_impalad_rpcz">
+
+ <title>RPC Services Page</title>
+
+ <conbody>
+
+ <p>
+ The <b>RPC durations</b> page of the <codeph>impalad</codeph> debug Web UI is at
+ <b>/rpcz</b> under the main <codeph>impalad</codeph> Web UI.
+ </p>
+
+ <p>
+ This page displays information, such as the duration, about the RPC communications of
+ this <codeph>impalad</codeph> with other Impala daemons.
+ </p>
+
+ </conbody>
+
</concept>
<concept id="webui_impalad_sessions">
@@ -333,17 +611,19 @@ under the License.
<conbody>
<p>
- By default, the <uicontrol>sessions</uicontrol> page of the debug web UI is at
- <codeph>http://<varname>impala-server-hostname</varname>:25000/sessions</codeph> (non-secure cluster) or
- <codeph>https://<varname>impala-server-hostname</varname>:25000/sessions</codeph> (secure cluster).
+ The <b>Sessions</b> page of the <codeph>impalad</codeph> debug Web UI is at
+ <b>/session</b> under the main <codeph>impalad</codeph> Web UI.
</p>
<p>
- This page displays information about the sessions currently connected to this <cmdname>impalad</cmdname>
- instance. For example, sessions could include connections from the <cmdname>impala-shell</cmdname>
- command, JDBC or ODBC applications, or the Impala Query UI in the Hue web interface.
+ This page displays information about the sessions currently connected to this
+ <cmdname>impalad</cmdname> instance. For example, sessions could include connections
+ from the <codeph>impala-shell</codeph> command, JDBC or ODBC applications, or the
+ Impala Query UI in the Hue web interface.
</p>
+
</conbody>
+
</concept>
<concept id="webui_impalad_threadz">
@@ -353,17 +633,18 @@ under the License.
<conbody>
<p>
- By default, the <uicontrol>threadz</uicontrol> page of the debug web UI is at
- <codeph>http://<varname>impala-server-hostname</varname>:25000/threadz</codeph> (non-secure cluster) or
- <codeph>https://<varname>impala-server-hostname</varname>:25000/threadz</codeph> (secure cluster).
+ The <b>Threads</b> page of the <codeph>impalad</codeph> debug Web UI is at
+ <b>/threadz</b> under the main <codeph>impalad</codeph> Web UI.
</p>
<p>
- This page displays information about the threads used by this instance of <cmdname>impalad</cmdname>,
- and shows which categories they are grouped into. Making use of this information requires substantial
- knowledge about Impala internals.
+ This page displays information about the threads used by this instance of
+ <codeph>impalad</codeph>, and it shows which categories they are grouped into. Making
+ use of this information requires substantial knowledge about Impala internals.
</p>
+
</conbody>
+
</concept>
<concept id="webui_impalad_varz">
@@ -373,43 +654,69 @@ under the License.
<conbody>
<p>
- By default, the <uicontrol>varz</uicontrol> page of the debug web UI is at
- <codeph>http://<varname>impala-server-hostname</varname>:25000/varz</codeph> (non-secure cluster) or
- <codeph>https://<varname>impala-server-hostname</varname>:25000/varz</codeph> (secure cluster).
+ The <b>Varz</b> page of the <codeph>impalad</codeph> debug Web UI is at <b>/varz</b>
+ under the main <codeph>impalad</codeph> Web UI.
</p>
<p>
- This page shows the configuration settings in effect when this instance of <cmdname>impalad</cmdname>
- communicates with other Hadoop components such as HDFS and YARN. These settings are collected from a set
- of configuration files; Impala might not actually make use of all settings.
+ This page shows the configuration settings in effect when this instance of
+ <codeph>impalad</codeph> communicates with other Hadoop components such as HDFS and
+ YARN. These settings are collected from a set of configuration files.
</p>
<p>
- The bottom of this page also lists all the command-line settings in effect for this instance of
- <cmdname>impalad</cmdname>. See <xref href="impala_config_options.xml#config_options"/> for information
+ The bottom of this page also lists all the command-line settings in effect for this
+ instance of <codeph>impalad</codeph>. See
+ <xref
+ href="impala_config_options.xml#config_options"/> for information
about modifying these values.
</p>
+
</conbody>
+
</concept>
+
</concept>
- <concept audience="hidden" id="webui_statestored">
+ <concept id="webui_statestored">
<title>Debug Web UI for statestored</title>
<conbody>
- <p></p>
+ <p>
+ To debug and troubleshoot the <codeph>statestored</codeph> daemon using a web-based
+ interface, open the URL
+ <codeph>http://<varname>impala‑server‑hostname</varname>:25010/</codeph> in
+ a browser. (For secure clusters, use the prefix <codeph>https://</codeph> instead of
+ <codeph>http://</codeph>.)
+ </p>
+
</conbody>
- <concept audience="hidden" id="webui_statestored_disabling">
+ <concept id="webui_statestored_disabling">
<title>Turning off the Web UI for statestored</title>
<conbody>
- <p></p>
+ <p>
+ To disable Web UI for the <codeph>statestored</codeph>:
+ </p>
+
+ <ol>
+ <li>
+ Stop the <codeph>statestored</codeph>.
+ </li>
+
+ <li>
+ Restart the <codeph>statestored</codeph> with the
+ <codeph>‑‑enable_webserver=false</codeph> startup flag.
+ </li>
+ </ol>
+
</conbody>
+
</concept>
<concept id="webui_statestored_main">
@@ -419,17 +726,38 @@ under the License.
<conbody>
<p>
- By default, the main page of the debug web UI is at
- <codeph>http://<varname>impala-server-hostname</varname>:25010/</codeph> (non-secure cluster) or
- <codeph>https://<varname>impala-server-hostname</varname>:25010/</codeph> (secure cluster).
+ The main <codeph>statestored</codeph> Web UI page at <b>/</b> lists the following
+ information about the <codeph>statestored</codeph>:
</p>
- <p>
- This page lists the version of the <cmdname>impalad</cmdname> daemon, plus basic hardware and software
- information about the corresponding host, such as information about the CPU, memory, disks, and operating
- system version.
- </p>
+ <ul>
+ <li>
+ The version of the <codeph>statestored</codeph> daemon
+ </li>
+
+ <li>
+ Process start time
+ </li>
+
+ <li>
+ Hardware information
+ </li>
+
+ <li>
+ OS information
+ </li>
+
+ <li>
+ Process information
+ </li>
+
+ <li>
+ CGroup information
+ </li>
+ </ul>
+
</conbody>
+
</concept>
<concept id="webui_statestored_logs">
@@ -439,19 +767,22 @@ under the License.
<conbody>
<p>
- By default, the <uicontrol>logs</uicontrol> page of the debug web UI is at
- <codeph>http://<varname>impala-server-hostname</varname>:25010/logs</codeph> (non-secure cluster) or
- <codeph>https://<varname>impala-server-hostname</varname>:25010/logs</codeph> (secure cluster).
+ The <b>INFO logs</b> page of the debug Web UI is at <b>/logs</b> under the main
+ <codeph>statestored</codeph> Web UI.
</p>
<p>
- This page shows the last portion of the <filepath>impalad.INFO</filepath> log file, the most detailed of
- the info, warning, and error logs for the <cmdname>impalad</cmdname> daemon. You can refer here to see
- the details of the most recent operations, whether the operations succeeded or encountered errors. This
- central page can be more convenient than looking around the filesystem for the log files, which could be
- in different locations on different hosts.
+ This page shows the last portion of the <filepath>statestored.INFO</filepath> log
+ file, including the info, warning, and error logs for the
+ <codeph>statestored</codeph>. You can refer here to see the details of the most recent
+ operations, whether the operations succeeded or encountered errors. This page provides
+ one central place for the log files and saves you from looking around the filesystem
+ for the log files, which could be in different locations on clusters that use cluster
+ management software.
</p>
+
</conbody>
+
</concept>
<concept id="webui_statestored_memz">
@@ -461,17 +792,18 @@ under the License.
<conbody>
<p>
- By default, the <uicontrol>memz</uicontrol> page of the debug web UI is at
- <codeph>http://<varname>impala-server-hostname</varname>:25010/memz</codeph> (non-secure cluster) or
- <codeph>https://<varname>impala-server-hostname</varname>:25010/memz</codeph> (secure cluster).
+ The <b>Memory Usage</b> page of the debug Web UI is at <b>/memz</b> under the main
+ <codeph>statestored</codeph> Web UI.
</p>
<p>
- This page displays summary and detailed information about memory usage by the <cmdname>impalad</cmdname>
- daemon. You can see the memory limit in effect for the node, and how much of that memory Impala is
- currently using.
+ This page displays summary and detailed information about memory usage by the
+ <codeph>statestored</codeph>. You can see the memory limit in effect for the node, and
+ how much of that memory Impala is currently using.
</p>
+
</conbody>
+
</concept>
<concept id="webui_statestored_metrics">
@@ -481,16 +813,37 @@ under the License.
<conbody>
<p>
- By default, the <uicontrol>metrics</uicontrol> page of the debug web UI is at
- <codeph>http://<varname>impala-server-hostname</varname>:25010/metrics</codeph> (non-secure cluster) or
- <codeph>https://<varname>impala-server-hostname</varname>:25010/metrics</codeph> (secure cluster).
+ The <b>Metrics</b> page of the debug Web UI is at <b>/metrics</b> under the main
+ <codeph>statestored</codeph> Web UI.
</p>
<p>
- This page displays the current set of metrics: counters and flags representing various aspects of
- <cmdname>impalad</cmdname> internal operation.
+ This page displays the current set of metrics: counters and flags representing various
+ aspects of <codeph>statestored</codeph> internal operation.
</p>
+
</conbody>
+
+ </concept>
+
+ <concept id="webui_statestored_rpcz">
+
+ <title>RPC Services Page</title>
+
+ <conbody>
+
+ <p>
+ The <b>RPC durations</b> page of the <codeph>statestored</codeph> debug Web UI is at
+ <b>/rpcz</b> under the main <codeph>statestored</codeph> Web UI.
+ </p>
+
+ <p>
+ This page displays information, such as the durations, about the RPC communications of
+ this <codeph>statestored</codeph> with other Impala daemons.
+ </p>
+
+ </conbody>
+
</concept>
<concept id="webui_statestored_subscribers">
@@ -500,15 +853,17 @@ under the License.
<conbody>
<p>
- By default, the <uicontrol>subscribers</uicontrol> page of the debug web UI is at
- <codeph>http://<varname>impala-server-hostname</varname>:25010/subscribers</codeph> (non-secure cluster)
- or <codeph>https://<varname>impala-server-hostname</varname>:25010/subscribers</codeph> (secure cluster).
+ The <b>Subscribers</b> page of the debug Web UI is at <b>/subscribers</b> under the
+ main <codeph>statestored</codeph> Web UI.
</p>
<p>
- This page displays information about...
+ This page displays information about the other Impala daemons that have registered
+ with the <codeph>statestored</codeph> to receive and send updates.
</p>
+
</conbody>
+
</concept>
<concept id="webui_statestored_threadz">
@@ -518,17 +873,18 @@ under the License.
<conbody>
<p>
- By default, the <uicontrol>threadz</uicontrol> page of the debug web UI is at
- <codeph>http://<varname>impala-server-hostname</varname>:25010/threadz</codeph> (non-secure cluster) or
- <codeph>https://<varname>impala-server-hostname</varname>:25010/threadz</codeph> (secure cluster).
+ The <b>Threads</b> page of the debug Web UI is at <b>/threadz</b> under the main
+ <codeph>statestored</codeph> Web UI.
</p>
<p>
- This page displays information about the threads used by this instance of <cmdname>impalad</cmdname>,
- and shows which categories they are grouped into. Making use of this information requires substantial
- knowledge about Impala internals.
+ This page displays information about the threads used by this instance of
+ <codeph>statestored</codeph>, and shows which categories they are grouped into. Making
+ use of this information requires substantial knowledge about Impala internals.
</p>
+
</conbody>
+
</concept>
<concept id="webui_statestored_topics">
@@ -538,15 +894,17 @@ under the License.
<conbody>
<p>
- By default, the <uicontrol>topics</uicontrol> page of the debug web UI is at
- <codeph>http://<varname>impala-server-hostname</varname>:25010/topics</codeph> (non-secure cluster) or
- <codeph>https://<varname>impala-server-hostname</varname>:25010/topics</codeph> (secure cluster).
+ The <b>Topics</b> page of the debug Web UI is at <b>/topics</b> under the main
+ <codeph>statestored</codeph> Web UI.
</p>
<p>
- This page displays information about...
+ This page displays information about the topics to which the other Impala daemons have
+ registered to receive updates.
</p>
+
</conbody>
+
</concept>
<concept id="webui_statestored_varz">
@@ -556,43 +914,69 @@ under the License.
<conbody>
<p>
- By default, the <uicontrol>varz</uicontrol> page of the debug web UI is at
- <codeph>http://<varname>impala-server-hostname</varname>:25010/varz</codeph> (non-secure cluster) or
- <codeph>https://<varname>impala-server-hostname</varname>:25010/varz</codeph> (secure cluster).
+ The <b>Varz</b> page of the debug Web UI is at <b>/varz</b> under the main
+ <codeph>statestored</codeph> Web UI.
</p>
<p>
- This page shows the configuration settings in effect when this instance of <cmdname>impalad</cmdname>
- communicates with other Hadoop components such as HDFS and YARN. These settings are collected from a set
- of configuration files; Impala might not actually make use of all settings.
+ This page shows the configuration settings in effect when this instance of
+ <codeph>statestored</codeph> communicates with other Hadoop components such as HDFS
+ and YARN. These settings are collected from a set of configuration files.
</p>
<p>
- The bottom of this page also lists all the command-line settings in effect for this instance of
- <cmdname>impalad</cmdname>. See <xref href="impala_config_options.xml#config_options"/> for information
+ The bottom of this page also lists all the command-line settings in effect for this
+ instance of <codeph>statestored</codeph>. See
+ <xref
+ href="impala_config_options.xml#config_options"/> for information
about modifying these values.
</p>
+
</conbody>
+
</concept>
+
</concept>
- <concept audience="hidden" id="webui_catalogd">
+ <concept id="webui_catalogd">
<title>Debug Web UI for catalogd</title>
<conbody>
- <p></p>
+ <p>
+ The main page of the debug Web UI is at
+ <codeph>http://<varname>impala‑server‑hostname</varname>:25020/</codeph>
+ (non-secure cluster) or
+ <codeph>https://<varname>impala‑server‑hostname</varname>:25020/</codeph>
+ (secure cluster).
+ </p>
+
</conbody>
- <concept audience="hidden" id="webui_catalogd_disabling">
+ <concept id="webui_catalogd_disabling">
- <title>Turning off the Web UI for impalad</title>
+ <title>Turning off the Web UI for catalogd</title>
<conbody>
- <p></p>
+ <p>
+ To disable Web UI for the <codeph>catalogd</codeph>:
+ </p>
+
+ <ol>
+ <li>
+ Stop the <codeph>catalogd</codeph>.
+ </li>
+
+ <li>
+ Restart the <codeph>catalogd</codeph> with the
+ <codeph>‑‑enable_webserver=false</codeph> startup flag.
+ </li>
+ </ol>
+
</conbody>
+
</concept>
<concept id="webui_catalogd_main">
@@ -602,17 +986,38 @@ under the License.
<conbody>
<p>
- By default, the main page of the debug web UI is at
- <codeph>http://<varname>impala-server-hostname</varname>:25020/</codeph> (non-secure cluster) or
- <codeph>https://<varname>impala-server-hostname</varname>:25020/</codeph> (secure cluster).
+ The main <codeph>catalogd</codeph> Web UI page at <b>/</b> lists the following
+ information about the <codeph>catalogd</codeph>:
</p>
- <p>
- This page lists the version of the <cmdname>impalad</cmdname> daemon, plus basic hardware and software
- information about the corresponding host, such as information about the CPU, memory, disks, and operating
- system version.
- </p>
+ <ul>
+ <li>
+ The version of the <codeph>catalogd</codeph> daemon
+ </li>
+
+ <li>
+ Process start time
+ </li>
+
+ <li>
+ Hardware information
+ </li>
+
+ <li>
+ OS information
+ </li>
+
+ <li>
+ Process information
+ </li>
+
+ <li>
+ CGroup information
+ </li>
+ </ul>
+
</conbody>
+
</concept>
<concept id="webui_catalogd_catalog">
@@ -622,18 +1027,60 @@ under the License.
<conbody>
<p>
- By default, the <uicontrol>catalog</uicontrol> page of the debug web UI is at
- <codeph>http://<varname>impala-server-hostname</varname>:25020/catalog</codeph> (non-secure cluster) or
- <codeph>https://<varname>impala-server-hostname</varname>:25020/catalog</codeph> (secure cluster).
+ The <b>Catalog</b> page of the debug Web UI is at <b>/catalog</b> under the main
+ <codeph>catalogd</codeph> Web UI.
+ </p>
+
+ <p>
+ This page displays a list of databases and associated tables recognized by this
+ instance of <codeph>catalogd</codeph>. You can use this page to locate which database
+ a table is in, check the exact spelling of a database or table name, look for
+ identical table names in multiple databases. The catalog information is represented as
+ the underlying Thrift data structures.
+ </p>
+
+ </conbody>
+
+ </concept>
+
+ <concept id="webui_catalogd_jmx">
+
+ <title>JMX</title>
+
+ <conbody>
+
+ <p>
+ The <b>JMX</b> page of the <codeph>catalogd</codeph> debug Web UI is at <b>/jmx</b>
+ under the main <codeph>catalogd</codeph> Web UI.
+ </p>
+
+ <p>
+ This page displays monitoring information about various JVM subsystems, such as memory
+ pools, thread management, runtime. etc.
+ </p>
+
+ </conbody>
+
+ </concept>
+
+ <concept id="webui_catalogd_log_level">
+
+ <title>Java Log Level</title>
+
+ <conbody>
+
+ <p>
+ The <b>Change log level</b> page of the <codeph>catalogd</codeph> debug Web UI is at
+ <b>/log_level</b> under the main <codeph>catalogd</codeph> Web UI.
</p>
<p>
- This page displays a list of databases and associated tables recognized by this instance of
- <cmdname>impalad</cmdname>. You can use this page to locate which database a table is in, check the
- exact spelling of a database or table name, look for identical table names in multiple databases, and so
- on.
+ The page displays the current Java and backend log levels and allows you to change the
+ log levels dynamically without having to restart the <codeph>catalogd</codeph>
</p>
+
</conbody>
+
</concept>
<concept id="webui_catalogd_logs">
@@ -643,19 +1090,43 @@ under the License.
<conbody>
<p>
- By default, the <uicontrol>logs</uicontrol> page of the debug web UI is at
- <codeph>http://<varname>impala-server-hostname</varname>:25020/logs</codeph> (non-secure cluster) or
- <codeph>https://<varname>impala-server-hostname</varname>:25020/logs</codeph> (secure cluster).
+ The <b>INFO logs</b> page of the debug Web UI is at <b>/logs</b> under the main
+ <codeph>catalogd</codeph> Web UI.
</p>
<p>
- This page shows the last portion of the <filepath>impalad.INFO</filepath> log file, the most detailed of
- the info, warning, and error logs for the <cmdname>impalad</cmdname> daemon. You can refer here to see
- the details of the most recent operations, whether the operations succeeded or encountered errors. This
- central page can be more convenient than looking around the filesystem for the log files, which could be
- in different locations on different hosts.
+ This page shows the last portion of the <codeph>catalogd.INFO</codeph> log file,
+ including the info, warning, and error logs for the <codeph>catalogd</codeph> daemon.
+ You can refer here to see the details of the most recent operations, whether the
+ operations succeeded or encountered errors. This page provides one central place for
+ the log files and saves you from looking around the filesystem for the log files,
+ which could be in different locations on clusters that use cluster management
+ software.
</p>
+
</conbody>
+
+ </concept>
+
+ <concept id="webui_catalogd_memz">
+
+ <title>Memz Page</title>
+
+ <conbody>
+
+ <p>
+ The <b>Memory Usage</b> page of the debug Web UI is at <b>/memz</b> under the main
+ <codeph>catalogd</codeph> Web UI.
+ </p>
+
+ <p>
+ This page displays summary and detailed information about memory usage by the
+ <codeph>catalogd</codeph>. You can see the memory limit in effect for the node, and
+ how much of that memory Impala is currently using.
+ </p>
+
+ </conbody>
+
</concept>
<concept id="webui_catalogd_metrics">
@@ -665,16 +1136,60 @@ under the License.
<conbody>
<p>
- By default, the <uicontrol>metrics</uicontrol> page of the debug web UI is at
- <codeph>http://<varname>impala-server-hostname</varname>:25020/metrics</codeph> (non-secure cluster) or
- <codeph>https://<varname>impala-server-hostname</varname>:25020/metrics</codeph> (secure cluster).
+ The <b>Metrics</b> page of the debug Web UI is at <b>/metrics</b> under the main
+ <codeph>catalogd</codeph> Web UI.
</p>
<p>
- This page displays the current set of metrics: counters and flags representing various aspects of
- <cmdname>impalad</cmdname> internal operation.
+ This page displays the current set of metrics: counters and flags representing various
+ aspects of <codeph>catalogd</codeph> internal operation.
</p>
+
+ <p/>
+
</conbody>
+
+ </concept>
+
+ <concept id="webui_catalogd_rpcz">
+
+ <title>RPC Services Page</title>
+
+ <conbody>
+
+ <p>
+ The <b>RPC durations</b> page of the <codeph>catalogd</codeph> debug Web UI is at
+ <b>/rpcz</b> under the main <codeph>catalogd</codeph> Web UI.
+ </p>
+
+ <p>
+ This page displays information, such as the durations, about the RPC communications of
+ this <codeph>catalogd</codeph> with other Impala daemons.
+ </p>
+
+ </conbody>
+
+ </concept>
+
+ <concept id="webui_catalogd_threadz">
+
+ <title>Threadz Page</title>
+
+ <conbody>
+
+ <p>
+ The <b>Threads</b> page of the debug Web UI is at <b>/threadz</b> under the main
+ <codeph>catalogd</codeph> Web UI.
+ </p>
+
+ <p>
+ This page displays information about the threads used by this instance of
+ <codeph>catalogd</codeph>, and shows which categories they are grouped into. Making
+ use of this information requires substantial knowledge about Impala internals.
+ </p>
+
+ </conbody>
+
</concept>
<concept id="webui_catalogd_varz">
@@ -684,23 +1199,28 @@ under the License.
<conbody>
<p>
- By default, the <uicontrol>varz</uicontrol> page of the debug web UI is at
- <codeph>http://<varname>impala-server-hostname</varname>:25020/varz</codeph> (non-secure cluster) or
- <codeph>https://<varname>impala-server-hostname</varname>:25020/varz</codeph> (secure cluster).
+ The <b>Varz</b> page of the debug Web UI is at <b>/varz</b> under the main
+ <codeph>catalogd</codeph> Web UI.
</p>
<p>
- This page shows the configuration settings in effect when this instance of <cmdname>impalad</cmdname>
- communicates with other Hadoop components such as HDFS and YARN. These settings are collected from a set
- of configuration files; Impala might not actually make use of all settings.
+ This page shows the configuration settings in effect when this instance of
+ <codeph>catalogd</codeph> communicates with other Hadoop components such as HDFS and
+ YARN. These settings are collected from a set of configuration files.
</p>
<p>
- The bottom of this page also lists all the command-line settings in effect for this instance of
- <cmdname>impalad</cmdname>. See <xref href="impala_config_options.xml#config_options"/> for information
+ The bottom of this page also lists all the command-line settings in effect for this
+ instance of <codeph>catalogd</codeph>. See
+ <xref
+ href="impala_config_options.xml#config_options"/> for information
about modifying these values.
</p>
+
</conbody>
+
</concept>
+
</concept>
+
</concept>
[impala] 06/06: IMPALA-7981: Add host disk statistics to profile
Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
arodoni pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 10e052a681863a08d895c5447c6c7bf8b31156c1
Author: Lars Volker <lv...@cloudera.com>
AuthorDate: Thu Mar 21 08:44:34 2019 -0700
IMPALA-7981: Add host disk statistics to profile
This change adds host disk statistics to profiles. For each host that
participates in the query execution it adds the read and write bandwidth
across all disks. This includes all data read or written by the host as
part of the execution of a query (spilling), by the HDFS data node, and
by other processes running on the same system.
The change adds tests for the added functionality to the backend and end
to end tests.
Change-Id: I373e7da47a0d722938e6ca1572c49a502951ed57
Reviewed-on: http://gerrit.cloudera.org:8080/12822
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/runtime/query-state.cc | 9 ++
be/src/util/disk-info.h | 6 ++
be/src/util/system-state-info-test.cc | 51 ++++++++++
be/src/util/system-state-info.cc | 175 ++++++++++++++++++++++-----------
be/src/util/system-state-info.h | 82 ++++++++++++++-
tests/query_test/test_observability.py | 7 +-
6 files changed, 266 insertions(+), 64 deletions(-)
diff --git a/be/src/runtime/query-state.cc b/be/src/runtime/query-state.cc
index 0d33a12..134b35e 100644
--- a/be/src/runtime/query-state.cc
+++ b/be/src/runtime/query-state.cc
@@ -170,6 +170,15 @@ Status QueryState::Init(const TExecQueryFInstancesParams& exec_rpc_params) {
"HostNetworkTx", TUnit::BYTES_PER_SECOND, [system_state_info] () {
return system_state_info->GetNetworkUsage().tx_rate;
});
+ // Add disk stats
+ host_profile_->AddChunkedTimeSeriesCounter(
+ "HostDiskReadThroughput", TUnit::BYTES_PER_SECOND, [system_state_info] () {
+ return system_state_info->GetDiskStats().read_rate;
+ });
+ host_profile_->AddChunkedTimeSeriesCounter(
+ "HostDiskWriteThroughput", TUnit::BYTES_PER_SECOND, [system_state_info] () {
+ return system_state_info->GetDiskStats().write_rate;
+ });
}
// Starting a new query creates threads and consumes a non-trivial amount of memory.
diff --git a/be/src/util/disk-info.h b/be/src/util/disk-info.h
index cb43b15..42c188e 100644
--- a/be/src/util/disk-info.h
+++ b/be/src/util/disk-info.h
@@ -64,6 +64,12 @@ class DiskInfo {
return disks_[disk_id].is_rotational;
}
+ /// Returns true if the disk with name 'device_name' is known to the DiskInfo class,
+ /// i.e. we consider it to be useful to include in metrics and such.
+ static bool is_known_disk(const std::string& device_name) {
+ return disk_name_to_disk_id_.find(device_name) != disk_name_to_disk_id_.end();
+ }
+
static std::string DebugString();
private:
diff --git a/be/src/util/system-state-info-test.cc b/be/src/util/system-state-info-test.cc
index c8680ba..a02a140 100644
--- a/be/src/util/system-state-info-test.cc
+++ b/be/src/util/system-state-info-test.cc
@@ -17,6 +17,7 @@
#include "common/atomic.h"
#include "testutil/gtest-util.h"
+#include "util/disk-info.h"
#include "util/system-state-info.h"
#include "util/time.h"
@@ -57,6 +58,16 @@ TEST_F(SystemStateInfoTest, ReadProcNetDev) {
EXPECT_GT(state[SystemStateInfo::NET_TX_PACKETS], 0);
}
+// Smoke test to make sure that we read non-zero values from /proc/diskstats.
+TEST_F(SystemStateInfoTest, ReadProcDiskStats) {
+ info.ReadCurrentProcDiskStats();
+ const SystemStateInfo::DiskValues& state = info.disk_values_[info.disk_val_idx_];
+ EXPECT_GT(state[SystemStateInfo::DISK_READS_COMPLETED], 0);
+ EXPECT_GT(state[SystemStateInfo::DISK_SECTORS_READ], 0);
+ EXPECT_GT(state[SystemStateInfo::DISK_WRITES_COMPLETED], 0);
+ EXPECT_GT(state[SystemStateInfo::DISK_SECTORS_WRITTEN], 0);
+}
+
// Tests parsing a line similar to the first line of /proc/stat.
TEST_F(SystemStateInfoTest, ParseProcStat) {
// Fields are: user nice system idle iowait irq softirq steal guest guest_nice
@@ -105,6 +116,24 @@ TEST_F(SystemStateInfoTest, ParseProcNetDevStringCentos6) {
EXPECT_EQ(state[SystemStateInfo::NET_TX_PACKETS], 84770);
}
+// Tests parsing a string similar to the contents of /proc/diskstats. This test also makes
+// sure that values of a partition are not accounted towards the disk it is on.
+TEST_F(SystemStateInfoTest, ParseProcDiskStatsString) {
+ // We cannot hard-code the device name since our Jenkins workers might see different
+ // device names in their virtual environments.
+ string disk_stats =
+ R"( 8 0 $0 17124835 222797 716029974 8414920 43758807 38432095 7867287712 630179264 0 32547524 638999340
+ 8 1 $01 17124482 222797 716027002 8414892 43546943 38432095 7867287712 629089180 0 31590972 637917344)";
+ string device = DiskInfo::device_name(0);
+ cout << Substitute(disk_stats, device);
+ info.ReadProcDiskStatsString(Substitute(disk_stats, device));
+ const SystemStateInfo::DiskValues& state = info.disk_values_[info.disk_val_idx_];
+ EXPECT_EQ(state[SystemStateInfo::DISK_SECTORS_READ], 716029974);
+ EXPECT_EQ(state[SystemStateInfo::DISK_READS_COMPLETED], 17124835);
+ EXPECT_EQ(state[SystemStateInfo::DISK_SECTORS_WRITTEN], 7867287712);
+ EXPECT_EQ(state[SystemStateInfo::DISK_WRITES_COMPLETED], 43758807);
+}
+
// Tests that computing CPU ratios doesn't overflow
TEST_F(SystemStateInfoTest, ComputeCpuRatiosIntOverflow) {
// Load old and new values for CPU counters. These values are from a system where we
@@ -167,5 +196,27 @@ TEST_F(SystemStateInfoTest, ComputeNetworkUsage) {
EXPECT_EQ(n.tx_rate, 12000);
}
+// Tests the computation logic for disk statistics.
+TEST_F(SystemStateInfoTest, ComputeDiskStats) {
+ // Two sets of values in the format of /proc/diskstats. We cannot hard-code the device
+ // name since our Jenkins workers might see different device names in their virtual
+ // environments.
+ string disk_stats_1 = R"( 1 15 ram15 0 0 0 0 0 0 0 0 0 0 0
+ 7 0 loop0 0 0 0 0 0 0 0 0 0 0 0
+ 8 0 $0 0 0 1000 0 0 0 2000 0 0 0 0
+ 8 1 $01 0 0 1000 0 0 0 2000 0 0 0 0)";
+ string disk_stats_2 = R"( 8 0 $0 0 0 2000 0 0 0 4000 0 0 0 0
+ 8 1 $01 0 0 2000 0 0 0 4000 0 0 0 0)";
+
+ string device = DiskInfo::device_name(0);
+ info.ReadProcDiskStatsString(Substitute(disk_stats_1, device));
+ info.ReadProcDiskStatsString(Substitute(disk_stats_2, device));
+ int period_ms = 500;
+ info.ComputeDiskStats(period_ms);
+ const SystemStateInfo::DiskStats& ds = info.GetDiskStats();
+ EXPECT_EQ(ds.read_rate, 2 * 1000 * SystemStateInfo::BYTES_PER_SECTOR);
+ EXPECT_EQ(ds.write_rate, 2 * 2000 * SystemStateInfo::BYTES_PER_SECTOR);
+}
+
} // namespace impala
diff --git a/be/src/util/system-state-info.cc b/be/src/util/system-state-info.cc
index 1194bf4..e808724 100644
--- a/be/src/util/system-state-info.cc
+++ b/be/src/util/system-state-info.cc
@@ -21,12 +21,13 @@
#include "kudu/util/env.h"
#include "kudu/util/faststring.h"
#include "kudu/util/logging.h"
+#include "util/disk-info.h"
#include "util/error-util.h"
#include "util/string-parser.h"
#include "util/system-state-info.h"
#include "util/time.h"
-#include <algorithm>
+#include <numeric>
#include <fstream>
#include <iostream>
@@ -39,43 +40,79 @@ using kudu::ReadFileToString;
using strings::Split;
using strings::SkipWhitespace;
+namespace {
+/// Reads a file into a buffer and returns whether the read was successful. If not
+/// successful, writes a warning into the log.
+bool ReadFileOrWarn(const string& path, faststring* buf) {
+ kudu::Status status = ReadFileToString(Env::Default(), path, buf);
+ if (!status.ok()) {
+ KLOG_EVERY_N_SECS(WARNING, 60) << "Could not open " << path << ": "
+ << status.message() << endl;
+ return false;
+ }
+ return true;
+}
+
+template <class T>
+void ReadIntArray(const StringPiece& sp, int expected_num_values, T* out) {
+ vector<StringPiece> counters = Split(sp, " ", SkipWhitespace());
+
+ // Something is wrong with the number of values that we read
+ if (expected_num_values > counters.size()) {
+ memset(out, 0, sizeof(*out));
+ KLOG_EVERY_N_SECS(WARNING, 60) << "Failed to parse enough values: expected "
+ << expected_num_values << " but found " << counters.size() << ". Input was: "
+ << sp;
+ return;
+ }
+
+ for (int i = 0; i < expected_num_values; ++i) {
+ int64_t* v = &(*out)[i];
+ if (!safe_strto64(counters[i].as_string(), v)) {
+ KLOG_EVERY_N_SECS(WARNING, 60) << "Failed to parse value: " << *v;
+ *v = 0;
+ }
+ DCHECK_GE(*v, 0) << "Value " << i << ": " << *v;
+ }
+}
+} // anonymous namespace
+
namespace impala {
// Partially initializing cpu_ratios_ will default-initialize the remaining members.
SystemStateInfo::SystemStateInfo() {
memset(&cpu_ratios_, 0, sizeof(cpu_ratios_));
memset(&network_usage_, 0, sizeof(network_usage_));
+ memset(&disk_stats_, 0, sizeof(disk_stats_));
ReadCurrentProcStat();
ReadCurrentProcNetDev();
last_net_update_ms_ = MonotonicMillis();
+ ReadCurrentProcDiskStats();
+ last_disk_update_ms_ = MonotonicMillis();
}
void SystemStateInfo::CaptureSystemStateSnapshot() {
// Capture and compute CPU usage
ReadCurrentProcStat();
ComputeCpuRatios();
+
int64_t now = MonotonicMillis();
ReadCurrentProcNetDev();
DCHECK_GT(last_net_update_ms_, 0L);
ComputeNetworkUsage(now - last_net_update_ms_);
last_net_update_ms_ = now;
-}
-int64_t SystemStateInfo::ParseInt64(const string& val) const {
- StringParser::ParseResult result;
- int64_t parsed = StringParser::StringToInt<int64_t>(val.c_str(), val.size(), &result);
- if (result == StringParser::PARSE_SUCCESS) return parsed;
- return -1;
+ now = MonotonicMillis();
+ ReadCurrentProcDiskStats();
+ DCHECK_GT(last_disk_update_ms_, 0L);
+ ComputeDiskStats(now - last_disk_update_ms_);
+ last_disk_update_ms_ = now;
}
void SystemStateInfo::ReadCurrentProcStat() {
string path = "/proc/stat";
faststring buf;
- kudu::Status status = ReadFileToString(Env::Default(), path, &buf);
- if (!status.ok()) {
- LOG(WARNING) << "Could not open " << path << ": " << status.message() << endl;
- return;
- }
+ if (!ReadFileOrWarn(path, &buf)) return;
StringPiece buf_sp(reinterpret_cast<const char*>(buf.data()), buf.size());
vector<StringPiece> lines = strings::Split(buf_sp, "\n");
DCHECK_GT(lines.size(), 0);
@@ -83,21 +120,14 @@ void SystemStateInfo::ReadCurrentProcStat() {
}
void SystemStateInfo::ReadProcStatString(const string& stat_string) {
- stringstream ss(stat_string);
-
- // Skip the first value ('cpu ');
- ss.ignore(5);
-
cpu_val_idx_ = 1 - cpu_val_idx_;
+ DCHECK(cpu_val_idx_ == 0 || cpu_val_idx_ == 1) << "cpu_val_idx_: " << cpu_val_idx_;
CpuValues& next_values = cpu_values_[cpu_val_idx_];
- for (int i = 0; i < NUM_CPU_VALUES; ++i) {
- int64_t v = -1;
- ss >> v;
- DCHECK_GE(v, 0) << "Value " << i << ": " << v;
- // Clamp invalid entries at 0
- next_values[i] = max(v, 0L);
- }
+ // Skip the first value ('cpu ');
+ StringPiece sp(stat_string);
+ DCHECK(sp.starts_with("cpu "));
+ ReadIntArray(sp.substr(5), NUM_CPU_VALUES, &next_values);
}
void SystemStateInfo::ComputeCpuRatios() {
@@ -126,27 +156,19 @@ void SystemStateInfo::ComputeCpuRatios() {
void SystemStateInfo::ReadCurrentProcNetDev() {
string path = "/proc/net/dev";
faststring buf;
- kudu::Status status = ReadFileToString(Env::Default(), path, &buf);
- if (!status.ok()) {
- LOG(WARNING) << "Could not open " << path << ": " << status.message() << endl;
- return;
- }
+ if (!ReadFileOrWarn(path, &buf)) return;
ReadProcNetDevString(buf.ToString());
}
void SystemStateInfo::ReadProcNetDevString(const string& dev_string) {
- stringstream ss(dev_string);
- string line;
-
- // Discard the first two lines that contain the header
- getline(ss, line);
- getline(ss, line);
-
net_val_idx_ = 1 - net_val_idx_;
+ DCHECK(net_val_idx_ == 0 || net_val_idx_ == 1) << "net_val_idx_: " << net_val_idx_;
NetworkValues& sum_values = network_values_[net_val_idx_];
memset(&sum_values, 0, sizeof(sum_values));
- while (getline(ss, line)) {
+ StringPiece sp(dev_string);
+ vector<StringPiece> lines = Split(sp, "\n");
+ for (const StringPiece& line : lines) {
NetworkValues line_values;
ReadProcNetDevLine(line, &line_values);
AddNetworkValues(line_values, &sum_values);
@@ -154,8 +176,8 @@ void SystemStateInfo::ReadProcNetDevString(const string& dev_string) {
}
void SystemStateInfo::ReadProcNetDevLine(
- const string& dev_string, NetworkValues* result) {
- StringPiece sp = StringPiece(dev_string);
+ const StringPiece& dev_string, NetworkValues* result) {
+ StringPiece sp(dev_string);
// Lines can have leading whitespace
StripWhiteSpace(&sp);
@@ -172,23 +194,7 @@ void SystemStateInfo::ReadProcNetDevLine(
return;
}
- vector<StringPiece> counters = Split(sp.substr(colon_idx + 1), " ", SkipWhitespace());
-
- // Something is wrong with the number of values that we read
- if (NUM_NET_VALUES > counters.size()) {
- KLOG_EVERY_N_SECS(WARNING, 60) << "Failed to parse enough values: expected "
- << NUM_NET_VALUES << " but found " << counters.size();
- return;
- }
-
- for (int i = 0; i < NUM_NET_VALUES; ++i) {
- int64_t* v = &(*result)[i];
- if (!safe_strto64(counters[i].as_string(), v)) {
- KLOG_EVERY_N_SECS(WARNING, 60) << "Failed to parse value: " << *v;
- *v = 0;
- }
- DCHECK_GE(*v, 0) << "Value " << i << ": " << *v;
- }
+ ReadIntArray(sp.substr(colon_idx + 1), NUM_NET_VALUES, result);
}
void SystemStateInfo::AddNetworkValues(const NetworkValues& a, NetworkValues* b) {
@@ -196,7 +202,7 @@ void SystemStateInfo::AddNetworkValues(const NetworkValues& a, NetworkValues* b)
}
void SystemStateInfo::ComputeNetworkUsage(int64_t period_ms) {
- if (period_ms == 0) return;
+ if (period_ms <= 0) return;
// Compute network throughput in bytes per second
const NetworkValues& cur = network_values_[net_val_idx_];
const NetworkValues& old = network_values_[1 - net_val_idx_];
@@ -207,4 +213,59 @@ void SystemStateInfo::ComputeNetworkUsage(int64_t period_ms) {
network_usage_.tx_rate = tx_bytes * 1000L / period_ms;
}
+void SystemStateInfo::ReadCurrentProcDiskStats() {
+ string path = "/proc/diskstats";
+ faststring buf;
+ if (!ReadFileOrWarn(path, &buf)) return;
+ ReadProcDiskStatsString(buf.ToString());
+}
+
+void SystemStateInfo::ReadProcDiskStatsString(const string& disk_stats) {
+ stringstream ss(disk_stats);
+ string line;
+
+ disk_val_idx_ = 1 - disk_val_idx_;
+ DCHECK(disk_val_idx_ == 0 || disk_val_idx_ == 1) << "disk_val_idx_: " << disk_val_idx_;
+ DiskValues& sum_values = disk_values_[disk_val_idx_];
+ memset(&sum_values, 0, sizeof(sum_values));
+
+ while (getline(ss, line)) {
+ DiskValues line_values;
+ ReadProcDiskStatsLine(line, &line_values);
+ AddDiskValues(line_values, &sum_values);
+ }
+}
+
+void SystemStateInfo::ReadProcDiskStatsLine(
+ const string& disk_stats, DiskValues* result) {
+ stringstream ss(disk_stats);
+
+ // Read the device IDs and name
+ string major, minor, disk_name;
+ ss >> major >> minor >> disk_name;
+
+ if (!DiskInfo::is_known_disk(disk_name)) {
+ memset(result, 0, sizeof(*result));
+ return;
+ }
+
+ ReadIntArray(StringPiece(disk_stats).substr(ss.tellg()), NUM_DISK_VALUES, result);
+}
+
+void SystemStateInfo::AddDiskValues(const DiskValues& a, DiskValues* b) {
+ for (int i = 0; i < NUM_DISK_VALUES; ++i) (*b)[i] += a[i];
+}
+
+void SystemStateInfo::ComputeDiskStats(int64_t period_ms) {
+ if (period_ms <= 0) return;
+ // Compute disk throughput in bytes per second
+ const DiskValues& cur = disk_values_[disk_val_idx_];
+ const DiskValues& old = disk_values_[1 - disk_val_idx_];
+ int64_t read_sectors = cur[DISK_SECTORS_READ] - old[DISK_SECTORS_READ];
+ disk_stats_.read_rate = read_sectors * BYTES_PER_SECTOR * 1000 / period_ms;
+
+ int64_t write_sectors = cur[DISK_SECTORS_WRITTEN] - old[DISK_SECTORS_WRITTEN];
+ disk_stats_.write_rate = write_sectors * BYTES_PER_SECTOR * 1000 / period_ms;
+}
+
} // namespace impala
diff --git a/be/src/util/system-state-info.h b/be/src/util/system-state-info.h
index e22d404..c17f562 100644
--- a/be/src/util/system-state-info.h
+++ b/be/src/util/system-state-info.h
@@ -20,10 +20,15 @@
#include <array>
#include <cstdint>
#include <string>
+#include <sstream>
#include <gtest/gtest_prod.h> // for FRIEND_TEST
#include "common/names.h"
+#include "common/logging.h"
+
+class StringPiece;
+
namespace impala {
/// Utility class to track and compute system resource consumption.
@@ -61,9 +66,17 @@ class SystemStateInfo {
/// calls to CaptureSystemStateSnapshot().
const NetworkUsage& GetNetworkUsage() { return network_usage_; }
- private:
- int64_t ParseInt64(const std::string& val) const;
+ /// Disk statistics rates in bytes per second
+ struct DiskStats {
+ int64_t read_rate;
+ int64_t write_rate;
+ };
+
+ /// Returns a struct containing the disk throughput for the interval between the last
+ /// two calls to CaptureSystemStateSnapshot().
+ const DiskStats& GetDiskStats() { return disk_stats_; }
+ private:
/// Rotates 'cpu_val_idx_' and reads the current CPU usage values from /proc/stat into
/// the current set of values.
void ReadCurrentProcStat();
@@ -126,7 +139,7 @@ class SystemStateInfo {
/// Two buffers to keep the current and previous set of network counter values.
NetworkValues network_values_[2];
/// Index into network_values_ that points to the current set of values. We maintain a
- /// separate index for CPU and network to be able to update them independently, e.g. in
+ /// separate index for network values to be able to update them independently, e.g. in
/// tests.
int net_val_idx_ = 0;
@@ -140,7 +153,7 @@ class SystemStateInfo {
/// Parses a single line as they appear in /proc/net/dev into 'result'. Entries are set
/// to 0 for the local loopback interface and for invalid entries.
- void ReadProcNetDevLine(const string& dev_string, NetworkValues* result);
+ void ReadProcNetDevLine(const StringPiece& dev_string, NetworkValues* result);
/// Computes b = b + a.
void AddNetworkValues(const NetworkValues& a, NetworkValues* b);
@@ -148,7 +161,7 @@ class SystemStateInfo {
/// Compute the network usage.
void ComputeNetworkUsage(int64_t period_ms);
- /// The compute network usage between the current and previous snapshots in
+ /// The network usage between the current and previous snapshots in
/// network_values_. Updated in ComputeNetworkUsage().
NetworkUsage network_usage_;
@@ -156,12 +169,71 @@ class SystemStateInfo {
/// CaptureSystemStateSnapshot().
int64_t last_net_update_ms_;
+ /// The enum names correspond to the fields of /proc/diskstats
+ /// https://www.kernel.org/doc/Documentation/iostats.txt
+ enum PROC_DISK_STAT_VALUES {
+ DISK_READS_COMPLETED,
+ DISK_READS_MERGED,
+ DISK_SECTORS_READ,
+ DISK_TIME_READING_MS,
+ DISK_WRITES_COMPLETED,
+ DISK_WRITES_MERGED,
+ DISK_SECTORS_WRITTEN,
+ DISK_TIME_WRITING_MS,
+ DISK_IO_IN_PROGRESS,
+ DISK_TIME_IN_IO_MS,
+ DISK_WEIGHTED_TIME_IN_IO_MS,
+ NUM_DISK_VALUES
+ };
+
+ /// Number of bytes per disk sector.
+ static const int BYTES_PER_SECTOR = 512;
+
+ /// We store these values in an array so that we can iterate over them, e.g. when
+ /// reading them from a file or summing them up.
+ typedef std::array<int64_t, NUM_DISK_VALUES> DiskValues;
+ /// Two buffers to keep the current and previous set of disk counter values.
+ DiskValues disk_values_[2];
+ /// Index into disk_values_ that points to the current set of values. We maintain a
+ /// separate index for disk values to be able to update them independently, e.g. in
+ /// tests.
+ int disk_val_idx_ = 0;
+
+ /// Rotates disk_val_idx_ and reads the current set of values from /proc/diskstats into
+ /// disk_values_.
+ void ReadCurrentProcDiskStats();
+
+ /// Rotates disk_val_idx_ and parses the content of 'disk_stats' into disk_values_.
+ /// disk_stats must be in the format of /proc/diskstats.
+ void ReadProcDiskStatsString(const string& disk_stats);
+
+ /// Parses a single line as they appear in /proc/diskstats into 'result'. Invalid
+ /// entries are set to 0.
+ void ReadProcDiskStatsLine(const string& disk_stats, DiskValues* result);
+
+ /// Computes b = b + a.
+ void AddDiskValues(const DiskValues& a, DiskValues* b);
+
+ /// Computes the read and write rate for the most recent period.
+ void ComputeDiskStats(int64_t period_ms);
+
+ /// The disk statistics between the current and previous snapshots in
+ /// disk_values_. Updated in ComputeDiskStats().
+ DiskStats disk_stats_;
+
+ /// The last time of reading disk stats values from /proc/diskstats. Used by
+ /// CaptureSystemStateSnapshot().
+ int64_t last_disk_update_ms_;
+
FRIEND_TEST(SystemStateInfoTest, ComputeCpuRatios);
FRIEND_TEST(SystemStateInfoTest, ComputeCpuRatiosIntOverflow);
+ FRIEND_TEST(SystemStateInfoTest, ComputeDiskStats);
FRIEND_TEST(SystemStateInfoTest, ComputeNetworkUsage);
+ FRIEND_TEST(SystemStateInfoTest, ParseProcDiskStatsString);
FRIEND_TEST(SystemStateInfoTest, ParseProcNetDevString);
FRIEND_TEST(SystemStateInfoTest, ParseProcNetDevStringCentos6);
FRIEND_TEST(SystemStateInfoTest, ParseProcStat);
+ FRIEND_TEST(SystemStateInfoTest, ReadProcDiskStats);
FRIEND_TEST(SystemStateInfoTest, ReadProcNetDev);
FRIEND_TEST(SystemStateInfoTest, ReadProcStat);
};
diff --git a/tests/query_test/test_observability.py b/tests/query_test/test_observability.py
index a1dca7f..fd4c796 100644
--- a/tests/query_test/test_observability.py
+++ b/tests/query_test/test_observability.py
@@ -441,6 +441,7 @@ class TestObservability(ImpalaTestSuite):
for line in profile.splitlines():
assert not re.search("HostCpu.*Percentage", line)
assert not re.search("HostNetworkRx", line)
+ assert not re.search("HostDiskReadThroughput", line)
def test_query_profile_contains_host_resource_metrics(self):
"""Tests that the query profile contains various CPU and network metrics."""
@@ -453,7 +454,9 @@ class TestObservability(ImpalaTestSuite):
"HostCpuSysPercentage (500.000ms):",
"HostCpuUserPercentage (500.000ms):",
"HostNetworkRx (500.000ms):",
- "HostNetworkTx (500.000ms):"]
+ "HostNetworkTx (500.000ms):",
+ "HostDiskReadThroughput (500.000ms):",
+ "HostDiskWriteThroughput (500.000ms):"]
# Assert that all expected counters exist in the profile.
for expected_str in expected_strs:
@@ -493,7 +496,7 @@ class TestObservability(ImpalaTestSuite):
result = self.execute_query("select sleep(2000)", query_opts)
thrift_profile = self._get_thrift_profile(result.query_id)
- expected_keys = ["HostCpuUserPercentage", "HostNetworkRx"]
+ expected_keys = ["HostCpuUserPercentage", "HostNetworkRx", "HostDiskReadThroughput"]
for key in expected_keys:
counters = self._find_ts_counters_in_thrift_profile(thrift_profile, key)
# The query will run on a single node, we will only find the counter once.
[impala] 01/06: IMPALA-8385: Refactor Sentry admin user check
Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
arodoni pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 284a9814aad89877fe6b80456c628cdaaa286b24
Author: Fredy Wijaya <fw...@cloudera.com>
AuthorDate: Mon Apr 8 17:00:16 2019 -0700
IMPALA-8385: Refactor Sentry admin user check
This patch moves the hardcoded Sentry admin user check from the generic
"show roles" and "show grant" catalog operations to Sentry authorization
plugin, i.e. SentryImpaladAuthorizationManager. This patch also removes
isAdmin() from AuthorizationManager interface.
Testing:
- Added a new authorization E2E test to test for Sentry admin check
- Ran all FE tests
- Ran all authorization E2E tests
Change-Id: I911228b09af7eed5d5dc002b20591ef64dc625d3
Reviewed-on: http://gerrit.cloudera.org:8080/12963
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/catalog/catalog-server.cc | 2 +-
be/src/catalog/catalog.cc | 7 +--
be/src/catalog/catalog.h | 7 +--
be/src/exec/catalog-op-executor.cc | 8 +--
be/src/exec/catalog-op-executor.h | 9 ++--
be/src/service/client-request-state.cc | 24 ---------
be/src/service/fe-support.cc | 29 +++++++++++
common/thrift/CatalogService.thrift | 6 ++-
.../impala/authorization/AuthorizationManager.java | 6 ---
.../authorization/NoneAuthorizationFactory.java | 5 --
.../ranger/RangerCatalogdAuthorizationManager.java | 5 --
.../sentry/SentryCatalogdAuthorizationManager.java | 6 ++-
.../sentry/SentryImpaladAuthorizationManager.java | 55 +++++++++++++++++---
.../java/org/apache/impala/service/FeSupport.java | 11 ++++
.../java/org/apache/impala/service/Frontend.java | 12 ++---
.../java/org/apache/impala/service/JniCatalog.java | 24 +++++----
tests/authorization/test_sentry.py | 60 ++++++++++++++++++++++
17 files changed, 194 insertions(+), 82 deletions(-)
diff --git a/be/src/catalog/catalog-server.cc b/be/src/catalog/catalog-server.cc
index 7952699..dee1871 100644
--- a/be/src/catalog/catalog-server.cc
+++ b/be/src/catalog/catalog-server.cc
@@ -208,7 +208,7 @@ class CatalogServiceThriftIf : public CatalogServiceIf {
void SentryAdminCheck(TSentryAdminCheckResponse& resp,
const TSentryAdminCheckRequest& req) override {
VLOG_RPC << "SentryAdminCheck(): request=" << ThriftDebugString(req);
- Status status = catalog_server_->catalog()->SentryAdminCheck(req);
+ Status status = catalog_server_->catalog()->SentryAdminCheck(req, &resp);
if (!status.ok()) LOG(ERROR) << status.GetDetail();
TStatus thrift_status;
status.ToThrift(&thrift_status);
diff --git a/be/src/catalog/catalog.cc b/be/src/catalog/catalog.cc
index c6becc3..585234a 100644
--- a/be/src/catalog/catalog.cc
+++ b/be/src/catalog/catalog.cc
@@ -61,7 +61,7 @@ Catalog::Catalog() {
{"getTableMetrics", "([B)Ljava/lang/String;", &get_table_metrics_id_},
{"getDbs", "([B)[B", &get_dbs_id_},
{"getFunctions", "([B)[B", &get_functions_id_},
- {"checkUserSentryAdmin", "([B)V", &sentry_admin_check_id_},
+ {"checkUserSentryAdmin", "([B)[B", &sentry_admin_check_id_},
{"getCatalogObject", "([B)[B", &get_catalog_object_id_},
{"getPartialCatalogObject", "([B)[B", &get_partial_catalog_object_id_},
{"getCatalogDelta", "([B)[B", &get_catalog_delta_id_},
@@ -184,8 +184,9 @@ Status Catalog::GetPartitionStats(
return JniUtil::CallJniMethod(catalog_, get_partition_stats_id_, req, resp);
}
-Status Catalog::SentryAdminCheck(const TSentryAdminCheckRequest& req) {
- return JniUtil::CallJniMethod(catalog_, sentry_admin_check_id_, req);
+Status Catalog::SentryAdminCheck(const TSentryAdminCheckRequest& req,
+ TSentryAdminCheckResponse* resp) {
+ return JniUtil::CallJniMethod(catalog_, sentry_admin_check_id_, req, resp);
}
Status Catalog::UpdateTableUsage(const TUpdateTableUsageRequest& req) {
diff --git a/be/src/catalog/catalog.h b/be/src/catalog/catalog.h
index 4096732..f1fe2e9 100644
--- a/be/src/catalog/catalog.h
+++ b/be/src/catalog/catalog.h
@@ -126,9 +126,10 @@ class Catalog {
const TGetPartitionStatsRequest& req, TGetPartitionStatsResponse* resp);
/// Checks whether the requesting user has admin privileges on the Sentry Service and
- /// returns OK if they do. Returns a bad status if the user is not an admin or if there
- /// was an error executing the request.
- Status SentryAdminCheck(const TSentryAdminCheckRequest& req);
+ /// returns OK if they do. Returns a bad status if there was an error executing the
+ /// request.
+ Status SentryAdminCheck(const TSentryAdminCheckRequest& req,
+ TSentryAdminCheckResponse* resp);
/// Update recently used table names and their use counts in an impalad since the last
/// report.
diff --git a/be/src/exec/catalog-op-executor.cc b/be/src/exec/catalog-op-executor.cc
index ce564d8..e820b0a 100644
--- a/be/src/exec/catalog-op-executor.cc
+++ b/be/src/exec/catalog-op-executor.cc
@@ -324,16 +324,16 @@ Status CatalogOpExecutor::GetPartitionStats(
return Status::OK();
}
-Status CatalogOpExecutor::SentryAdminCheck(const TSentryAdminCheckRequest& req) {
+Status CatalogOpExecutor::SentryAdminCheck(const TSentryAdminCheckRequest& req,
+ TSentryAdminCheckResponse* result) {
const TNetworkAddress& address =
MakeNetworkAddress(FLAGS_catalog_service_host, FLAGS_catalog_service_port);
Status cnxn_status;
CatalogServiceConnection client(env_->catalogd_client_cache(), address, &cnxn_status);
RETURN_IF_ERROR(cnxn_status);
- TSentryAdminCheckResponse resp;
RETURN_IF_ERROR(
- client.DoRpc(&CatalogServiceClientWrapper::SentryAdminCheck, req, &resp));
- return Status(resp.status);
+ client.DoRpc(&CatalogServiceClientWrapper::SentryAdminCheck, req, result));
+ return Status::OK();
}
Status CatalogOpExecutor::UpdateTableUsage(const TUpdateTableUsageRequest& req,
diff --git a/be/src/exec/catalog-op-executor.h b/be/src/exec/catalog-op-executor.h
index 15d5c5b..8284d85 100644
--- a/be/src/exec/catalog-op-executor.h
+++ b/be/src/exec/catalog-op-executor.h
@@ -79,10 +79,11 @@ class CatalogOpExecutor {
Status UpdateTableUsage(const TUpdateTableUsageRequest& req,
TUpdateTableUsageResponse* resp);
- /// Makes an RPC to the CatalogServer to verify whether the specified user has privileges
- /// to access the Sentry Policy Service. Returns OK if the user has privileges or
- /// a bad status if the user does not have privileges (or if there was an error).
- Status SentryAdminCheck(const TSentryAdminCheckRequest& re);
+ /// Makes an RPC to the CatalogServer to verify whether the specified user has
+ /// privileges / to access the Sentry Policy Service. Returns OK if the RPC was
+ /// successful, otherwise a bad status will be returned.
+ Status SentryAdminCheck(const TSentryAdminCheckRequest& req,
+ TSentryAdminCheckResponse* resp);
/// Set in Exec(), returns a pointer to the TDdlExecResponse of the DDL execution.
/// If called before Exec(), this will return NULL. Only set if the
diff --git a/be/src/service/client-request-state.cc b/be/src/service/client-request-state.cc
index f8c1b95..5e667fd 100644
--- a/be/src/service/client-request-state.cc
+++ b/be/src/service/client-request-state.cc
@@ -354,18 +354,6 @@ Status ClientRequestState::ExecLocalCatalogOp(
}
case TCatalogOpType::SHOW_ROLES: {
const TShowRolesParams& params = catalog_op.show_roles_params;
- if (params.is_admin_op) {
- // Verify the user has privileges to perform this operation by checking against
- // the Sentry Service (via the Catalog Server).
- catalog_op_executor_.reset(new CatalogOpExecutor(exec_env_, frontend_,
- server_profile_));
-
- TSentryAdminCheckRequest req;
- req.__set_header(TCatalogServiceRequestHeader());
- req.header.__set_requesting_user(effective_user());
- RETURN_IF_ERROR(catalog_op_executor_->SentryAdminCheck(req));
- }
-
// If we have made it here, the user has privileges to execute this operation.
// Return the results.
TShowRolesResult result;
@@ -375,18 +363,6 @@ Status ClientRequestState::ExecLocalCatalogOp(
}
case TCatalogOpType::SHOW_GRANT_PRINCIPAL: {
const TShowGrantPrincipalParams& params = catalog_op.show_grant_principal_params;
- if (params.is_admin_op) {
- // Verify the user has privileges to perform this operation by checking against
- // the Sentry Service (via the Catalog Server).
- catalog_op_executor_.reset(new CatalogOpExecutor(exec_env_, frontend_,
- server_profile_));
-
- TSentryAdminCheckRequest req;
- req.__set_header(TCatalogServiceRequestHeader());
- req.header.__set_requesting_user(effective_user());
- RETURN_IF_ERROR(catalog_op_executor_->SentryAdminCheck(req));
- }
-
TResultSet response;
RETURN_IF_ERROR(frontend_->GetPrincipalPrivileges(params, &response));
// Set the result set and its schema from the response.
diff --git a/be/src/service/fe-support.cc b/be/src/service/fe-support.cc
index 4a25250..5f16b86 100644
--- a/be/src/service/fe-support.cc
+++ b/be/src/service/fe-support.cc
@@ -549,6 +549,30 @@ Java_org_apache_impala_service_FeSupport_NativeUpdateTableUsage(
return result_bytes;
}
+// Calls the catalog server to to check if the given user is a Sentry admin.
+extern "C"
+JNIEXPORT jbyteArray JNICALL
+Java_org_apache_impala_service_FeSupport_NativeSentryAdminCheck(
+ JNIEnv* env, jclass caller_class, jbyteArray thrift_struct) {
+ TSentryAdminCheckRequest request;
+ THROW_IF_ERROR_RET(DeserializeThriftMsg(env, thrift_struct, &request), env,
+ JniUtil::internal_exc_class(), nullptr);
+
+ CatalogOpExecutor catalog_op_executor(ExecEnv::GetInstance(), nullptr, nullptr);
+ TSentryAdminCheckResponse result;
+ Status status = catalog_op_executor.SentryAdminCheck(request, &result);
+ if (!status.ok()) {
+ LOG(ERROR) << status.GetDetail();
+ status.AddDetail("Error making an RPC call to Catalog server.");
+ status.SetTStatus(&result);
+ }
+
+ jbyteArray result_bytes = nullptr;
+ THROW_IF_ERROR_RET(SerializeThriftMsg(env, &result, &result_bytes), env,
+ JniUtil::internal_exc_class(), result_bytes);
+ return result_bytes;
+}
+
// Calls in to the catalog server to request partial information about a
// catalog object.
extern "C"
@@ -663,6 +687,11 @@ static JNINativeMethod native_methods[] = {
(void*)::Java_org_apache_impala_service_FeSupport_NativeUpdateTableUsage
},
{
+ const_cast<char*>("NativeSentryAdminCheck"),
+ const_cast<char*>("([B)[B"),
+ (void*)::Java_org_apache_impala_service_FeSupport_NativeSentryAdminCheck
+ },
+ {
const_cast<char*>("NativeParseQueryOptions"),
const_cast<char*>("(Ljava/lang/String;[B)[B"),
(void*)::Java_org_apache_impala_service_FeSupport_NativeParseQueryOptions
diff --git a/common/thrift/CatalogService.thrift b/common/thrift/CatalogService.thrift
index 0f186a2..cd5b775 100644
--- a/common/thrift/CatalogService.thrift
+++ b/common/thrift/CatalogService.thrift
@@ -478,9 +478,11 @@ struct TSentryAdminCheckRequest {
}
struct TSentryAdminCheckResponse {
- // Contains an error if the user does not have privileges to access the Sentry Service
- // or if the Sentry Service is unavailable. Returns OK if the operation was successful.
+ // Returns OK if the operation was successful.
1: optional Status.TStatus status
+
+ // Returns true if the user is a Sentry admin user.
+ 2: required bool is_admin
}
struct TTableUsage {
diff --git a/fe/src/main/java/org/apache/impala/authorization/AuthorizationManager.java b/fe/src/main/java/org/apache/impala/authorization/AuthorizationManager.java
index 0f42345..e8ad5b9 100644
--- a/fe/src/main/java/org/apache/impala/authorization/AuthorizationManager.java
+++ b/fe/src/main/java/org/apache/impala/authorization/AuthorizationManager.java
@@ -36,12 +36,6 @@ import org.apache.impala.thrift.TShowRolesResult;
*/
public interface AuthorizationManager {
/**
- * Returns true if the given user is an admin user.
- * TODO: Do we really need this?
- */
- boolean isAdmin(User user) throws ImpalaException;
-
- /**
* Creates a role.
*/
void createRole(User requestingUser, TCreateDropRoleParams params,
diff --git a/fe/src/main/java/org/apache/impala/authorization/NoneAuthorizationFactory.java b/fe/src/main/java/org/apache/impala/authorization/NoneAuthorizationFactory.java
index f21752c..f347b60 100644
--- a/fe/src/main/java/org/apache/impala/authorization/NoneAuthorizationFactory.java
+++ b/fe/src/main/java/org/apache/impala/authorization/NoneAuthorizationFactory.java
@@ -72,11 +72,6 @@ public class NoneAuthorizationFactory implements AuthorizationFactory {
public static class NoneAuthorizationManager implements AuthorizationManager {
@Override
- public boolean isAdmin(User user) throws ImpalaException {
- return false;
- }
-
- @Override
public void createRole(User requestingUser, TCreateDropRoleParams params,
TDdlExecResponse response) throws ImpalaException {
throw new UnsupportedOperationException(String.format("%s is not supported",
diff --git a/fe/src/main/java/org/apache/impala/authorization/ranger/RangerCatalogdAuthorizationManager.java b/fe/src/main/java/org/apache/impala/authorization/ranger/RangerCatalogdAuthorizationManager.java
index cedd1e8..22c4b04 100644
--- a/fe/src/main/java/org/apache/impala/authorization/ranger/RangerCatalogdAuthorizationManager.java
+++ b/fe/src/main/java/org/apache/impala/authorization/ranger/RangerCatalogdAuthorizationManager.java
@@ -54,11 +54,6 @@ public class RangerCatalogdAuthorizationManager implements AuthorizationManager
}
@Override
- public boolean isAdmin(User user) throws ImpalaException {
- return false;
- }
-
- @Override
public void createRole(User requestingUser, TCreateDropRoleParams params,
TDdlExecResponse response) throws ImpalaException {
throw new UnsupportedOperationException(String.format(
diff --git a/fe/src/main/java/org/apache/impala/authorization/sentry/SentryCatalogdAuthorizationManager.java b/fe/src/main/java/org/apache/impala/authorization/sentry/SentryCatalogdAuthorizationManager.java
index 872dc63..ed9108c 100644
--- a/fe/src/main/java/org/apache/impala/authorization/sentry/SentryCatalogdAuthorizationManager.java
+++ b/fe/src/main/java/org/apache/impala/authorization/sentry/SentryCatalogdAuthorizationManager.java
@@ -71,8 +71,10 @@ public class SentryCatalogdAuthorizationManager implements AuthorizationManager
catalog_ = catalog;
}
- @Override
- public boolean isAdmin(User user) throws ImpalaException {
+ /**
+ * Checks if the given user is a Sentry admin.
+ */
+ public boolean isSentryAdmin(User user) throws ImpalaException {
return catalog_.getSentryProxy().isSentryAdmin(user);
}
diff --git a/fe/src/main/java/org/apache/impala/authorization/sentry/SentryImpaladAuthorizationManager.java b/fe/src/main/java/org/apache/impala/authorization/sentry/SentryImpaladAuthorizationManager.java
index 73cc11c..932a267 100644
--- a/fe/src/main/java/org/apache/impala/authorization/sentry/SentryImpaladAuthorizationManager.java
+++ b/fe/src/main/java/org/apache/impala/authorization/sentry/SentryImpaladAuthorizationManager.java
@@ -17,22 +17,30 @@
package org.apache.impala.authorization.sentry;
+import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.hadoop.hive.metastore.api.PrincipalType;
import org.apache.impala.authorization.AuthorizationChecker;
+import org.apache.impala.authorization.AuthorizationException;
import org.apache.impala.authorization.AuthorizationManager;
import org.apache.impala.authorization.User;
import org.apache.impala.catalog.Role;
import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.InternalException;
+import org.apache.impala.common.JniUtil;
import org.apache.impala.service.FeCatalogManager;
+import org.apache.impala.service.FeSupport;
+import org.apache.impala.thrift.TCatalogServiceRequestHeader;
import org.apache.impala.thrift.TCreateDropRoleParams;
import org.apache.impala.thrift.TDdlExecResponse;
+import org.apache.impala.thrift.TErrorCode;
import org.apache.impala.thrift.TGrantRevokePrivParams;
import org.apache.impala.thrift.TGrantRevokeRoleParams;
import org.apache.impala.thrift.TResultSet;
+import org.apache.impala.thrift.TSentryAdminCheckRequest;
+import org.apache.impala.thrift.TSentryAdminCheckResponse;
import org.apache.impala.thrift.TShowGrantPrincipalParams;
import org.apache.impala.thrift.TShowRolesParams;
import org.apache.impala.thrift.TShowRolesResult;
@@ -71,12 +79,6 @@ public class SentryImpaladAuthorizationManager implements AuthorizationManager {
}
@Override
- public boolean isAdmin(User user) throws ImpalaException {
- throw new UnsupportedOperationException(String.format(
- "%s is not supported in Impalad", ClassUtil.getMethodName()));
- }
-
- @Override
public void createRole(User requestingUser, TCreateDropRoleParams params,
TDdlExecResponse response) throws ImpalaException {
throw new UnsupportedOperationException(String.format(
@@ -92,6 +94,10 @@ public class SentryImpaladAuthorizationManager implements AuthorizationManager {
@Override
public TShowRolesResult getRoles(TShowRolesParams params) throws ImpalaException {
+ if (params.isIs_admin_op()) {
+ validateSentryAdmin(params.getRequesting_user());
+ }
+
TShowRolesResult result = new TShowRolesResult();
List<Role> roles = Lists.newArrayList();
if (params.isIs_show_current_roles() || params.isSetGrant_group()) {
@@ -180,6 +186,10 @@ public class SentryImpaladAuthorizationManager implements AuthorizationManager {
@Override
public TResultSet getPrivileges(TShowGrantPrincipalParams params)
throws ImpalaException {
+ if (params.isIs_admin_op()) {
+ validateSentryAdmin(params.getRequesting_user());
+ }
+
switch (params.getPrincipal_type()) {
case USER:
Set<String> groupNames = authzChecker_.get().getUserGroups(
@@ -210,4 +220,37 @@ public class SentryImpaladAuthorizationManager implements AuthorizationManager {
throw new UnsupportedOperationException(String.format(
"%s is not supported in Impalad", ClassUtil.getMethodName()));
}
+
+ /**
+ * Validates if the given user is a Sentry admin. The Sentry admin check will make an
+ * RPC call to the Catalog server. This check is necessary because some operations
+ * in this class does not need to make a call to Sentry, e.g. "show roles" and
+ * "show grant" because the authorization data can be retrieved directly from the
+ * Impalad catalog without going to Sentry. In order to ensure those operations can
+ * only be executed by a Sentry admin, a separate call to the Catalog server is needed
+ * to check if the given user is a Sentry admin.
+ *
+ * @throws AuthorizationException thrown when a given user is not a Sentry admin.
+ */
+ private static void validateSentryAdmin(String user) throws ImpalaException {
+ TSentryAdminCheckRequest request = new TSentryAdminCheckRequest();
+ TCatalogServiceRequestHeader header = new TCatalogServiceRequestHeader();
+ header.setRequesting_user(user);
+ request.setHeader(header);
+
+ byte[] thriftReq = JniUtil.serializeToThrift(request);
+ byte[] thriftRes = FeSupport.CheckSentryAdmin(thriftReq);
+
+ TSentryAdminCheckResponse response = new TSentryAdminCheckResponse();
+ JniUtil.deserializeThrift(response, thriftRes);
+
+ if (response.getStatus().getStatus_code() != TErrorCode.OK) {
+ throw new InternalException(String.format("Error requesting SentryAdminCheck: %s",
+ Joiner.on("\n").join(response.getStatus().getError_msgs())));
+ }
+ if (!response.isIs_admin()) {
+ throw new AuthorizationException(String.format("User '%s' does not have " +
+ "privileges to access the requested policy metadata.", user));
+ }
+ }
}
diff --git a/fe/src/main/java/org/apache/impala/service/FeSupport.java b/fe/src/main/java/org/apache/impala/service/FeSupport.java
index 36ef823..685b509 100644
--- a/fe/src/main/java/org/apache/impala/service/FeSupport.java
+++ b/fe/src/main/java/org/apache/impala/service/FeSupport.java
@@ -114,6 +114,9 @@ public class FeSupport {
public native static byte[] NativeUpdateTableUsage(byte[] thriftReq);
+ // Does an RPC to the Catalog Server to check if the given user is a Sentry admin.
+ public native static byte[] NativeSentryAdminCheck(byte[] thriftReq);
+
// Parses a string of comma-separated key=value query options ('csvQueryOptions'),
// updates the existing query options ('queryOptions') with them and returns the
// resulting serialized TQueryOptions object.
@@ -412,6 +415,14 @@ public class FeSupport {
return NativeGetPartialCatalogObject(thriftReq);
}
+ public static byte[] CheckSentryAdmin(byte[] thriftReq) {
+ try {
+ return NativeSentryAdminCheck(thriftReq);
+ } catch (UnsatisfiedLinkError e) {
+ loadLibrary();
+ }
+ return NativeSentryAdminCheck(thriftReq);
+ }
/**
* This function should be called explicitly by the FeSupport to ensure that
diff --git a/fe/src/main/java/org/apache/impala/service/Frontend.java b/fe/src/main/java/org/apache/impala/service/Frontend.java
index f43e959..f142245 100644
--- a/fe/src/main/java/org/apache/impala/service/Frontend.java
+++ b/fe/src/main/java/org/apache/impala/service/Frontend.java
@@ -24,9 +24,7 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
-import java.util.Random;
import java.util.Set;
-import java.util.UUID;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@@ -509,11 +507,13 @@ public class Frontend {
// Check if the user is part of the group (case-sensitive) this SHOW ROLE
// statement is targeting. If they are already a member of the group,
// the admin requirement can be removed.
+ // If the the statement is SHOW CURRENT ROLES, the admin requirement can also be
+ // removed.
Preconditions.checkState(ddl.getShow_roles_params().isSetIs_admin_op());
- if (ddl.getShow_roles_params().isSetGrant_group() &&
- groupNames.contains(ddl.getShow_roles_params().getGrant_group())) {
- ddl.getShow_roles_params().setIs_admin_op(false);
- }
+ ddl.getShow_roles_params().setIs_admin_op(!(
+ (ddl.getShow_roles_params().isSetGrant_group() &&
+ groupNames.contains(ddl.getShow_roles_params().getGrant_group())) ||
+ ddl.getShow_roles_params().isIs_show_current_roles()));
metadata.setColumns(Arrays.asList(
new TColumn("role_name", Type.STRING.toThrift())));
} else if (analysis.isShowGrantPrincipalStmt()) {
diff --git a/fe/src/main/java/org/apache/impala/service/JniCatalog.java b/fe/src/main/java/org/apache/impala/service/JniCatalog.java
index 00c28d6..eae7fd1 100644
--- a/fe/src/main/java/org/apache/impala/service/JniCatalog.java
+++ b/fe/src/main/java/org/apache/impala/service/JniCatalog.java
@@ -26,10 +26,10 @@ import java.util.List;
import java.util.UUID;
import org.apache.impala.authorization.AuthorizationConfig;
-import org.apache.impala.authorization.AuthorizationException;
import org.apache.impala.authorization.AuthorizationFactory;
import org.apache.impala.authorization.NoneAuthorizationFactory;
import org.apache.impala.authorization.User;
+import org.apache.impala.authorization.sentry.SentryCatalogdAuthorizationManager;
import org.apache.impala.catalog.CatalogException;
import org.apache.impala.catalog.CatalogServiceCatalog;
import org.apache.impala.catalog.Db;
@@ -60,6 +60,7 @@ import org.apache.impala.thrift.TLogLevel;
import org.apache.impala.thrift.TPrioritizeLoadRequest;
import org.apache.impala.thrift.TResetMetadataRequest;
import org.apache.impala.thrift.TSentryAdminCheckRequest;
+import org.apache.impala.thrift.TSentryAdminCheckResponse;
import org.apache.impala.thrift.TStatus;
import org.apache.impala.thrift.TUniqueId;
import org.apache.impala.thrift.TUpdateCatalogRequest;
@@ -298,20 +299,21 @@ public class JniCatalog {
}
/**
- * Verifies whether the user is configured as an admin. Throws an AuthorizationException
- * if the user does not have admin privileges.
- *
- * TODO: rename this method name.
+ * Verifies whether the user is configured as a Sentry admin.
*/
- public void checkUserSentryAdmin(byte[] thriftReq) throws ImpalaException,
- TException {
+ public byte[] checkUserSentryAdmin(byte[] thriftReq)
+ throws ImpalaException, TException {
TSentryAdminCheckRequest request = new TSentryAdminCheckRequest();
JniUtil.deserializeThrift(protocolFactory_, request, thriftReq);
+ TSerializer serializer = new TSerializer(protocolFactory_);
User user = new User(request.getHeader().getRequesting_user());
- if (!catalogOpExecutor_.getAuthzManager().isAdmin(user)) {
- throw new AuthorizationException(String.format("User '%s' does not have " +
- "privileges to access the requested policy metadata.", user.getName()));
- }
+ Preconditions.checkState(catalogOpExecutor_.getAuthzManager() instanceof
+ SentryCatalogdAuthorizationManager);
+
+ TSentryAdminCheckResponse response = new TSentryAdminCheckResponse();
+ response.setIs_admin(((SentryCatalogdAuthorizationManager)
+ catalogOpExecutor_.getAuthzManager()).isSentryAdmin(user));
+ return serializer.serialize(response);
}
/**
diff --git a/tests/authorization/test_sentry.py b/tests/authorization/test_sentry.py
new file mode 100644
index 0000000..494fdb5
--- /dev/null
+++ b/tests/authorization/test_sentry.py
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import pytest
+import os
+from getpass import getuser
+
+from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
+
+SENTRY_CONFIG_DIR = os.getenv('IMPALA_HOME') + '/fe/src/test/resources/'
+SENTRY_CONFIG_FILE = SENTRY_CONFIG_DIR + 'sentry-site.xml'
+
+
+class TestSentry(CustomClusterTestSuite):
+ """This class contains Sentry specific authorization tests."""
+ @pytest.mark.execute_serially
+ @CustomClusterTestSuite.with_args(
+ impalad_args="--server_name=server1 --sentry_config={0}".format(SENTRY_CONFIG_FILE),
+ catalogd_args="--sentry_config={0}".format(SENTRY_CONFIG_FILE))
+ def test_sentry_admin(self, unique_role, unique_name):
+ """
+ Tests that only admin user can execute certain authorization statements.
+ """
+ admin = getuser()
+ non_admin = unique_name
+ admin_client = self.create_impala_client()
+ non_admin_client = self.create_impala_client()
+ try:
+ self.execute_query_expect_success(admin_client,
+ "create role {0}".format(unique_role),
+ user=admin)
+
+ # show current roles is always allowed.
+ self.execute_query_expect_success(admin_client, "show current roles", user=admin)
+ self.execute_query_expect_success(non_admin_client, "show current roles",
+ user=non_admin)
+
+ for statement in ["show roles",
+ "show grant role {0}".format(unique_role)]:
+ self.execute_query_expect_success(admin_client, statement, user=admin)
+ result = self.execute_query_expect_failure(non_admin_client, statement,
+ user=non_admin)
+ assert "AuthorizationException: User '{0}' does not have privileges to access " \
+ "the requested policy metadata.".format(non_admin) in str(result)
+ finally:
+ admin_client.execute("drop role {0}".format(unique_role))
[impala] 04/06: IMPALA-8391: [DOCS] ALTER TABLE RENAME now renames
the underlying Kudu TABLE
Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
arodoni pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 65837f82e19be13367edeb1dbb6daec442189817
Author: Alex Rodoni <ar...@cloudera.com>
AuthorDate: Tue Apr 9 16:29:18 2019 -0700
IMPALA-8391: [DOCS] ALTER TABLE RENAME now renames the underlying Kudu TABLE
Change-Id: I704a0623a05166e5f636b6c1063e9feffb58e65b
Reviewed-on: http://gerrit.cloudera.org:8080/12975
Reviewed-by: Alex Rodoni <ar...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
docs/topics/impala_tables.xml | 59 +++++++++++++++++++++++--------------------
1 file changed, 32 insertions(+), 27 deletions(-)
diff --git a/docs/topics/impala_tables.xml b/docs/topics/impala_tables.xml
index 0da7955..b2b6a71 100644
--- a/docs/topics/impala_tables.xml
+++ b/docs/topics/impala_tables.xml
@@ -283,44 +283,49 @@ under the License.
<data name="Category" value="Kudu"/>
</metadata>
</prolog>
-
<conbody>
<p>
- Tables stored in Apache Kudu are treated specially, because Kudu manages its data independently of HDFS files.
- Some information about the table is stored in the metastore database for use by Impala. Other table metadata is
- managed internally by Kudu.
+ Tables stored in Apache Kudu are treated specially, because Kudu
+ manages its data independently of HDFS files. Some information about the
+ table is stored in the metastore database for use by Impala. Other table
+ metadata is managed internally by Kudu.
</p>
-
- <p> When you create a Kudu table through Impala, it is assigned an
+ <p>
+ When you create a Kudu table through Impala, the table is assigned an
internal Kudu table name of the form
<codeph>impala::<varname>db_name</varname>.<varname>table_name</varname></codeph>.
You can see the Kudu-assigned name in the output of <codeph>DESCRIBE
FORMATTED</codeph>, in the <codeph>kudu.table_name</codeph> field of
- the table properties. The Kudu-assigned name remains the same even if
- you use <codeph>ALTER TABLE</codeph> to rename the Impala table or move
- it to a different Impala database. You can issue the statement
- <codeph>ALTER TABLE <varname>impala_name</varname> SET
- TBLPROPERTIES('kudu.table_name' =
- '<varname>different_kudu_table_name</varname>')</codeph> for the
- external tables created with the <codeph>CREATE EXTERNAL TABLE</codeph>
- statement. Changing the <codeph>kudu.table_name</codeph> property of an
- external table switches which underlying Kudu table the Impala table
- refers to. The underlying Kudu table must already exist. </p>
-
- <p>In practice, external tables are typically used to access underlying
+ the table properties.
+ </p>
+ <p> For Impala-Kudu managed tables, <codeph>ALTER TABLE ...
+ RENAME</codeph> renames both the Impala and the Kudu table. </p>
+ <p> For Impala-Kudu external tables, <codeph>ALTER TABLE ...
+ RENAME</codeph> renames just the Impala table. To change the Kudu
+ table that an Impala external table points to, use <codeph>ALTER TABLE
+ <varname>impala_name</varname> SET TBLPROPERTIES('kudu.table_name' =
+ '<varname>different_kudu_table_name</varname>')</codeph>. The
+ underlying Kudu table must already exist. </p>
+ <p>
+ In practice, external tables are typically used to access underlying
Kudu tables that were created outside of Impala, that is, through the
- Kudu API. </p>
-
+ Kudu API.
+ </p>
<p>
- The <codeph>SHOW TABLE STATS</codeph> output for a Kudu table shows Kudu-specific details about the layout of the table.
- Instead of information about the number and sizes of files, the information is divided by the Kudu tablets.
- For each tablet, the output includes the fields
- <codeph># Rows</codeph> (although this number is not currently computed), <codeph>Start Key</codeph>, <codeph>Stop Key</codeph>, <codeph>Leader Replica</codeph>, and <codeph># Replicas</codeph>.
- The output of <codeph>SHOW COLUMN STATS</codeph>, illustrating the distribution of values within each column, is the same for Kudu tables
+ The <codeph>SHOW TABLE STATS</codeph> output for a Kudu table shows
+ Kudu-specific details about the layout of the table. Instead of
+ information about the number and sizes of files, the information is
+ divided by the Kudu tablets. For each tablet, the output includes the
+ fields <codeph># Rows</codeph> (although this number is not currently
+ computed), <codeph>Start Key</codeph>, <codeph>Stop Key</codeph>,
+ <codeph>Leader Replica</codeph>, and <codeph># Replicas</codeph>. The
+ output of <codeph>SHOW COLUMN STATS</codeph>, illustrating the
+ distribution of values within each column, is the same for Kudu tables
as for HDFS-backed tables.
</p>
-
- <p conref="../shared/impala_common.xml#common/kudu_internal_external_tables"/>
+ <p
+ conref="../shared/impala_common.xml#common/kudu_internal_external_tables"
+ />
</conbody>
</concept>