You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2016/07/14 19:05:12 UTC
[15/18] incubator-impala git commit: IMPALA-1671: Print time and link
to coordinator web UI once query is submitted in shell
IMPALA-1671: Print time and link to coordinator web UI once query is submitted in shell
To help supportability and debugging, it's helpful to have the impala
shell print out the coordinator time and the link to the coordinator
web UI once the query is submitted.
This is done by calling the PingImpalaService() routine everytime a
query is submitted, which returns the coordinator's hostname,
webserver port and the coordinator epoch time at that moment which the
shell then formats and prints out.
Added tests to verify these new messages.
Change-Id: I704eb64546e27c367830120241311fea6091266b
Reviewed-on: http://gerrit.cloudera.org:8080/3507
Reviewed-by: Sailesh Mukil <sa...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/900f1480
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/900f1480
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/900f1480
Branch: refs/heads/master
Commit: 900f148078b4fdb8375e3a3107d62fb3ca74e24f
Parents: 144cc68
Author: Sailesh Mukil <sa...@cloudera.com>
Authored: Mon Jun 27 12:59:34 2016 -0700
Committer: Taras Bobrovytsky <ta...@apache.org>
Committed: Thu Jul 14 19:04:45 2016 +0000
----------------------------------------------------------------------
be/src/service/impala-beeswax-server.cc | 2 ++
be/src/util/webserver.cc | 5 +++++
be/src/util/webserver.h | 3 +++
common/thrift/ImpalaService.thrift | 6 ++++++
shell/impala_client.py | 8 +++++---
shell/impala_shell.py | 23 ++++++++++++++++++-----
tests/shell/test_shell_commandline.py | 10 ++++++++++
7 files changed, 49 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/900f1480/be/src/service/impala-beeswax-server.cc
----------------------------------------------------------------------
diff --git a/be/src/service/impala-beeswax-server.cc b/be/src/service/impala-beeswax-server.cc
index 988386d..f7c1dc4 100644
--- a/be/src/service/impala-beeswax-server.cc
+++ b/be/src/service/impala-beeswax-server.cc
@@ -498,6 +498,8 @@ void ImpalaServer::PingImpalaService(TPingImpalaServiceResp& return_val) {
VLOG_RPC << "PingImpalaService()";
return_val.version = GetVersionString(true);
+ return_val.webserver_address = ExecEnv::GetInstance()->webserver()->Url();
+ return_val.epoch_time = reinterpret_cast<int64_t>(std::time(0));
VLOG_RPC << "PingImpalaService(): return_val=" << ThriftDebugString(return_val);
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/900f1480/be/src/util/webserver.cc
----------------------------------------------------------------------
diff --git a/be/src/util/webserver.cc b/be/src/util/webserver.cc
index 318a189..b514c36 100644
--- a/be/src/util/webserver.cc
+++ b/be/src/util/webserver.cc
@@ -215,6 +215,11 @@ bool Webserver::IsSecure() const {
return !FLAGS_webserver_certificate_file.empty();
}
+string Webserver::Url() {
+ return Substitute("$0://$1:$2", IsSecure() ? "https" : "http",
+ http_address_.hostname, http_address_.port);
+}
+
Status Webserver::Start() {
LOG(INFO) << "Starting webserver on " << http_address_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/900f1480/be/src/util/webserver.h
----------------------------------------------------------------------
diff --git a/be/src/util/webserver.h b/be/src/util/webserver.h
index 347fd94..014267f 100644
--- a/be/src/util/webserver.h
+++ b/be/src/util/webserver.h
@@ -87,6 +87,9 @@ class Webserver {
/// True if serving all traffic over SSL, false otherwise
bool IsSecure() const;
+ /// Returns the URL to the webserver as a string.
+ string Url();
+
private:
/// Contains all information relevant to rendering one Url. Each Url has one callback
/// that produces the output to render. The callback either produces a Json document
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/900f1480/common/thrift/ImpalaService.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/ImpalaService.thrift b/common/thrift/ImpalaService.thrift
index 0777c32..218a339 100644
--- a/common/thrift/ImpalaService.thrift
+++ b/common/thrift/ImpalaService.thrift
@@ -240,6 +240,12 @@ struct TInsertResult {
struct TPingImpalaServiceResp {
// The Impala service's version string.
1: string version
+
+ // The Impalad's webserver address.
+ 2: string webserver_address
+
+ // The Impalad's server time.
+ 3: i64 epoch_time
}
// Parameters for a ResetTable request which will invalidate a table's metadata.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/900f1480/shell/impala_client.py
----------------------------------------------------------------------
diff --git a/shell/impala_client.py b/shell/impala_client.py
index aa5e9fc..63195f0 100755
--- a/shell/impala_client.py
+++ b/shell/impala_client.py
@@ -229,10 +229,12 @@ class ImpalaClient(object):
self.transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.imp_service = ImpalaService.Client(protocol)
- result = self.imp_service.PingImpalaService()
- server_version = result.version
+ result = self.ping_impala_service()
self.connected = True
- return server_version
+ return result.version
+
+ def ping_impala_service(self):
+ return self.imp_service.PingImpalaService()
def close_connection(self):
"""Close the transport if it's still open"""
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/900f1480/shell/impala_shell.py
----------------------------------------------------------------------
diff --git a/shell/impala_shell.py b/shell/impala_shell.py
index deb65d8..eda4e05 100755
--- a/shell/impala_shell.py
+++ b/shell/impala_shell.py
@@ -676,9 +676,7 @@ class ImpalaShell(cmd.Cmd):
def _connect(self):
try:
- server_version = self.imp_client.connect()
- if server_version:
- self.server_version = server_version
+ self.server_version = self.imp_client.connect()
except TApplicationException:
# We get a TApplicationException if the transport is valid,
# but the RPC does not exist.
@@ -859,13 +857,28 @@ class ImpalaShell(cmd.Cmd):
The execution time is printed and the query is closed if it hasn't been already
"""
+
+ self._print_if_verbose("Query: %s" % (query.query,))
+ # TODO: Clean up this try block and refactor it (IMPALA-3814)
try:
- self._print_if_verbose("Query: %s" % (query.query,))
- start_time = time.time()
+ # Get the hostname, webserver port and the current coordinator time.
+ coordinator = self.imp_client.ping_impala_service()
+ # If the coordinator is on a different time zone, the epoch time returned will be
+ # different than from this system, so time.localtime(coordinator.epoch_time) will
+ # return the timestamp of the server.
+ self._print_if_verbose("Query submitted at: %s (Coordinator: %s)" % (time.strftime(
+ "%Y-%m-%d %H:%M:%S", time.localtime(coordinator.epoch_time)),
+ coordinator.webserver_address))
+ start_time = time.time()
self.last_query_handle = self.imp_client.execute_query(query)
self.query_handle_closed = False
self.last_summary = time.time()
+ if coordinator.webserver_address:
+ self._print_if_verbose(
+ "Query progress can be monitored at: %s/query_plan?query_id=%s" %
+ (coordinator.webserver_address, self.last_query_handle.id))
+
wait_to_finish = self.imp_client.wait_to_finish(self.last_query_handle,
self._periodic_wait_callback)
# Reset the progress stream.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/900f1480/tests/shell/test_shell_commandline.py
----------------------------------------------------------------------
diff --git a/tests/shell/test_shell_commandline.py b/tests/shell/test_shell_commandline.py
index f14e3ec..fa4fc44 100644
--- a/tests/shell/test_shell_commandline.py
+++ b/tests/shell/test_shell_commandline.py
@@ -417,3 +417,13 @@ class TestImpalaShell(ImpalaTestSuite):
% (os.path.join(QUERY_FILE_PATH, 'test_var_substitution.sql'))
result = run_impala_shell_cmd(args, expect_success=True)
assert_var_substitution(result)
+
+ def test_query_start_time_message(self):
+ results = run_impala_shell_cmd('--query="select 1"')
+ assert "Query submitted at: " in results.stderr
+
+ def test_query_coordinator_link_message(self):
+ results = run_impala_shell_cmd('--query="select 1"')
+ assert "(Coordinator: " in results.stderr
+ assert "Query progress can be monitored at: " in results.stderr
+